From 06599a037f859c2b5b4bd504e4626bd15d37d1f7 Mon Sep 17 00:00:00 2001
From: Mickael <mckmonster@mailo.com>
Date: Wed, 1 Jan 2020 19:52:21 +0100
Subject: [PATCH] Fix generation of stats

---
 scripts/common/charthelper.py         | 15 ++++++++++++---
 scripts/generate_chat_stat_daily.py   | 25 ++++++++++++++++++++-----
 scripts/generate_chat_stat_monthly.py | 23 ++++++++++++++++++-----
 3 files changed, 50 insertions(+), 13 deletions(-)

diff --git a/scripts/common/charthelper.py b/scripts/common/charthelper.py
index f4aa794..2fc47a1 100644
--- a/scripts/common/charthelper.py
+++ b/scripts/common/charthelper.py
@@ -16,11 +16,19 @@ def createElement(label, color, data):
       "data": data
   }
 
+
 def setTopicsInfo(topics, messagesDataTopic, id, length):
   for topic in topics:
-    if topic not in messagesDataTopic:
-      messagesDataTopic[topic] = [0] * 30
-    messagesDataTopic[topic][id] += length
+    result = None
+    if len(messagesDataTopic) > 0:
+      result = next((x for x in messagesDataTopic if x['label'] == topic), None)
+    
+    if result is None:
+      result = createElement(topic, getColor(), [0] * 30)
+      messagesDataTopic.append(result)
+
+    result['data'][id] += length
+
 
 def getTopics(channel):
   if 'description' in channel:
@@ -30,5 +38,6 @@ def getTopics(channel):
     return topics
   return ['global']
 
+
 if __name__ == "__main__":
   print("not executable")
diff --git a/scripts/generate_chat_stat_daily.py b/scripts/generate_chat_stat_daily.py
index 2e0781e..06a843b 100644
--- a/scripts/generate_chat_stat_daily.py
+++ b/scripts/generate_chat_stat_daily.py
@@ -8,6 +8,7 @@ from rocketchat_API.rocketchat import RocketChat
 import json
 import os
 import random
+import time
 from datetime import datetime, timedelta
 from common.rocketchathelper import getAllChannels, getAllMessages, Connection
 from common.savehelper import save
@@ -25,7 +26,7 @@ def main():
   labels = [None] * nbElements
   messagesByChannel = []
   usersByChannel = []
-  messagesByTopic = {}
+  messagesByTopic = []
   usersGlobal = []
 
   now = datetime.now()
@@ -42,7 +43,14 @@ def main():
 
   uniqueUserGlobal = [None] * nbElements
 
-  for channel in getAllChannels(rocket):
+  waittest = 0
+  idChannel = 0
+  channels = getAllChannels(rocket)
+  for channel in channels:
+    idChannel += 1
+    if channel['name'].startswith("cohorte"):
+      continue
+
     dataMess = []
     dataUsers = []
     print( channel['name'] )
@@ -52,6 +60,10 @@ def main():
     topics = getTopics(channel)    
     print(topics)
     messages = getAllMessages(rocket, channel['_id'], begindate= begin, enddate= end, count= channel["msgs"])
+    if messages is None:
+      print("ERROR to get messages")
+      break
+
     end = begin + timedelta(1)
 
     for id in range(0, nbElements):
@@ -60,9 +72,6 @@ def main():
       labels[id] = begin.strftime("%x")
       print("\t{0}".format(labels[id]))
       
-      if messages is None:
-        continue
-
       resultMess = list(filter(filter_message, messages))
       length = len(resultMess)
       dataMess.append(length)
@@ -88,6 +97,12 @@ def main():
 
     messagesByChannel.append(messageByChannel)
     usersByChannel.append(userByChannel)
+    waittest += 1
+    if waittest >= 5:
+      waittest = 0
+      print("Wait {}/{}".format(idChannel, len(channels)))
+      time.sleep(30)
+
 
   for id in range(0, nbElements):
     uniqueUserGlobal[id] = len(set(uniqueUserGlobal[id]))
diff --git a/scripts/generate_chat_stat_monthly.py b/scripts/generate_chat_stat_monthly.py
index afb2501..5de32e2 100644
--- a/scripts/generate_chat_stat_monthly.py
+++ b/scripts/generate_chat_stat_monthly.py
@@ -8,6 +8,7 @@ from rocketchat_API.rocketchat import RocketChat
 import json
 import os
 import random
+import time
 from datetime import datetime
 from monthdelta import monthdelta
 from common.rocketchathelper import getAllChannels, getAllMessages, Connection
@@ -26,7 +27,7 @@ def main():
   labels = [None] * 12
   messagesByChannel = []
   usersByChannel = []
-  messagesByTopic = {}
+  messagesByTopic = []
   usersGlobal = []
 
   now = datetime.now()
@@ -43,7 +44,14 @@ def main():
 
   uniqueUserGlobal = [None] * 12
 
-  for channel in getAllChannels(rocket):
+  waittest = 0
+  idChannel = 0
+  channels = getAllChannels(rocket)
+  for channel in channels:
+    idChannel += 1
+    if channel['name'].startswith("cohorte"):
+      continue
+
     dataMess = []
     dataUsers = []
     print( channel['name'] )
@@ -51,6 +59,9 @@ def main():
     end = begin + monthdelta(1)
 
     messages = getAllMessages(rocket, channel['_id'], begindate= begin, enddate=date, count= channel['msgs'])
+    if messages is None:
+      print("ERROR to get messages")
+      break
 
     topics = getTopics(channel)    
     print(topics)
@@ -60,9 +71,6 @@ def main():
       labels[id] = begin.strftime("%b %Y")
       print("\t{0}".format(labels[id]))
 
-      if messages is None:
-        continue
-      
       resultMess = list(filter(lambda mess: begin < datetime.strptime(mess["ts"], "%Y-%m-%dT%H:%M:%S.%fZ") < end, messages))
       length = len(resultMess)
       dataMess.append(length)
@@ -88,6 +96,11 @@ def main():
 
     messagesByChannel.append(messageByChannel)
     usersByChannel.append(userByChannel)
+    waittest += 1
+    if waittest >= 5:
+      print("Wait {}/{}".format(idChannel, len(channels)))
+      waittest = 0
+      time.sleep(30)
 
   for id in range(0, 12):
     uniqueUserGlobal[id] = len(set(uniqueUserGlobal[id]))