From f7baf818e1b9406652eb323e03e12185f636a5ee Mon Sep 17 00:00:00 2001 From: Mickael Date: Thu, 4 Jul 2019 21:15:35 +0200 Subject: [PATCH] refacto --- script/common/channelhelper.py | 17 +++- script/getchannels.py | 97 ++++++++++------------ script/getstats.py | 145 ++++++++++++++++----------------- 3 files changed, 133 insertions(+), 126 deletions(-) diff --git a/script/common/channelhelper.py b/script/common/channelhelper.py index 5be16e4..2fe3b9d 100644 --- a/script/common/channelhelper.py +++ b/script/common/channelhelper.py @@ -36,4 +36,19 @@ class Tsunami: PROJECT = 1 << 1 DEMOCRACY = 1 << 2 ECOLOGY = 1 << 3 - TECHNOLOGY = 1 << 4 \ No newline at end of file + TECHNOLOGY = 1 << 4 + +def getAllChannels(rocket): + index = 0 + allChannels = [] + while True: + channels = rocket.channels_list(offset= index).json() + + allChannels.extend([ channel for channel in channels['channels'] if 'archived' not in channel]) + if channels['count'] + channels['offset'] >= channels['total']: + break + index += channels['count'] + return allChannels + +if __name__ == "__main__": + print("Ce fichier est juste une librarie") \ No newline at end of file diff --git a/script/getchannels.py b/script/getchannels.py index 25291be..f56ea25 100644 --- a/script/getchannels.py +++ b/script/getchannels.py @@ -9,40 +9,35 @@ import json import dev_config as cfg import os import re -from common.channelhelper import getNodesOrigin +from common.channelhelper import getNodesOrigin, getAllChannels, Tsunami -colorInfo = { - 'global': 'orange', - 'technologie': 'gray', - 'democratie': 'red', - 'ecologie': 'green', - 'project': 'blue' -} +def main(): + + colorInfo = { + 'global': 'orange', + 'technologie': 'gray', + 'democratie': 'red', + 'ecologie': 'green', + 'project': 'blue' + } -rocket = RocketChat(cfg.rocket['user'], cfg.rocket['password'], server_url='https://coa.crapaud-fou.org') + rocket = RocketChat(cfg.rocket['user'], cfg.rocket['password'], server_url=cfg.rocket['server']) -sizebase = 100 -datas = [] -datas.append( { 'data':{'id':'global', 'label': 'global', 'size': sizebase, 'color': colorInfo['global'], 'href': 'https://coa.crapaud-fou.org/'}}) -datas.append( { 'data':{'id':'ecologie', 'label': 'ecologie', 'size': sizebase, 'color': colorInfo['ecologie'], 'href': 'https://coa.crapaud-fou.org/'}}) -datas.append( { 'data':{'id':'democratie', 'label': 'democratie', 'size': sizebase, 'color': colorInfo['democratie'], 'href': 'https://coa.crapaud-fou.org/'}}) -datas.append( { 'data':{'id':'technologie', 'label': 'technologie', 'size': sizebase, 'color': colorInfo['technologie'], 'href': 'https://coa.crapaud-fou.org/'}}) -datas.append( { 'data':{'id':'project', 'label': 'projet', 'size': sizebase, 'color': colorInfo['project'], 'href': 'https://coa.crapaud-fou.org/'}}) + sizebase = 100 + datas = [] + datas.append( { 'data':{'id':'global', 'label': 'global', 'size': sizebase, 'color': colorInfo['global'], 'href': 'https://coa.crapaud-fou.org/'}}) + datas.append( { 'data':{'id':'ecologie', 'label': 'ecologie', 'size': sizebase, 'color': colorInfo['ecologie'], 'href': 'https://coa.crapaud-fou.org/'}}) + datas.append( { 'data':{'id':'democratie', 'label': 'democratie', 'size': sizebase, 'color': colorInfo['democratie'], 'href': 'https://coa.crapaud-fou.org/'}}) + datas.append( { 'data':{'id':'technologie', 'label': 'technologie', 'size': sizebase, 'color': colorInfo['technologie'], 'href': 'https://coa.crapaud-fou.org/'}}) + datas.append( { 'data':{'id':'project', 'label': 'projet', 'size': sizebase, 'color': colorInfo['project'], 'href': 'https://coa.crapaud-fou.org/'}}) -cohortes = { 'fr': { 'updateMap': 'france_fr'}} -cohortescolor = { 'fr': 'green' } -index = 0 -nbChannels = 0 -nbCohorte = 0 -totalChannels = 0 -while True: - channels = rocket.channels_list(offset= index).json() - totalChannels = channels['total'] - - for channel in channels['channels']: + cohortes = { 'fr': { 'updateMap': 'france_fr'}} + cohortescolor = { 'fr': 'green' } + nbChannels = 0 + nbCohorte = 0 + totalChannels = 0 + for channel in getAllChannels(rocket): print("{}".format(channel['name'])) - if ('archived' in channel) and channel['archived']: - continue if channel['name'].find('cohorte') != -1: if 'description' in channel: @@ -64,7 +59,7 @@ while True: 'label': channel['fname'] if 'fname' in channel else channel['name'], 'size': size, 'color': 'grey', - 'href': 'https://coa.crapaud-fou.org/channel/'+channel['name'] + 'href': f"{cfg.rocket['server']}/channel/{channel['name']}" } } datas.append(node) @@ -76,31 +71,29 @@ while True: nbChannels += 1 - if channels['count'] + channels['offset'] >= channels['total']: - break - index += channels['count'] + # Récupération du répertoire racine du repo + rootFolder = os.path.join(os.path.dirname(__file__), '..') + # Répertoire pour stocker le fichier de sortie + dataFolder = os.path.join(rootFolder, 'public','data') + # Faut il essayer de le créer au cas ou? + # os.makedirs(dataFolderPath, exist_ok=True) + channelsFilePath = os.path.abspath(os.path.join(dataFolder,'channelslist.json')) -# Récupération du répertoire racine du repo -rootFolder = os.path.join(os.path.dirname(__file__), '..') -# Répertoire pour stocker le fichier de sortie -dataFolder = os.path.join(rootFolder, 'public','data') -# Faut il essayer de le créer au cas ou? -# os.makedirs(dataFolderPath, exist_ok=True) -channelsFilePath = os.path.abspath(os.path.join(dataFolder,'channelslist.json')) + #print("Ecriture dans : "+channelsFilePath) -#print("Ecriture dans : "+channelsFilePath) + with open(channelsFilePath, "w") as file_write: + json.dump(datas, file_write) -with open(channelsFilePath, "w") as file_write: - json.dump(datas, file_write) + cohortecolorFilePath = os.path.abspath(os.path.join(dataFolder,'cohortescolor.json')) + with open(cohortecolorFilePath, "w") as file_write: + json.dump(cohortescolor, file_write) -cohortecolorFilePath = os.path.abspath(os.path.join(dataFolder,'cohortescolor.json')) -with open(cohortecolorFilePath, "w") as file_write: - json.dump(cohortescolor, file_write) + cohorteFilePath = os.path.abspath(os.path.join(dataFolder,'cohorteslist.json')) + with open(cohorteFilePath, "w") as file_write: + json.dump(cohortes, file_write) -cohorteFilePath = os.path.abspath(os.path.join(dataFolder,'cohorteslist.json')) -with open(cohorteFilePath, "w") as file_write: - json.dump(cohortes, file_write) + pprint("Nb displayed channels : " + str(nbChannels)) + pprint("Nb cohorte channels : " + str(nbCohorte)) -pprint("Nb displayed channels : " + str(nbChannels)) -pprint("Nb cohorte channels : " + str(nbCohorte)) -pprint("Nb total channels : " + str(totalChannels)) +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/script/getstats.py b/script/getstats.py index 468c417..99744f6 100644 --- a/script/getstats.py +++ b/script/getstats.py @@ -3,7 +3,7 @@ # toutes les chaines sont en unicode (même les docstrings) from __future__ import unicode_literals -from pprint import pprint +# from pprint import pprint from rocketchat_API.rocketchat import RocketChat import json import dev_config as cfg @@ -11,8 +11,7 @@ import os import random from datetime import datetime from monthdelta import monthdelta -from common.channelhelper import getTsunamy -from common.channelhelper import Tsunami +from common.channelhelper import getTsunamy, Tsunami, getAllChannels def getColor(): r = random.randrange(255) @@ -38,19 +37,31 @@ def createElement(label, color, data) : "data": data } +def setTsunamyInfo(tsunamy, messagesDataTsunamy, id, length): + if tsunamy & Tsunami.GLOBAL: + messagesDataTsunamy[Tsunami.GLOBAL][id] += length + if tsunamy & Tsunami.PROJECT: + messagesDataTsunamy[Tsunami.PROJECT][id] += length + if tsunamy & Tsunami.DEMOCRACY: + messagesDataTsunamy[Tsunami.DEMOCRACY][id] += length + if tsunamy & Tsunami.ECOLOGY: + messagesDataTsunamy[Tsunami.ECOLOGY][id] += length + if tsunamy & Tsunami.TECHNOLOGY: + messagesDataTsunamy[Tsunami.TECHNOLOGY][id] += length + def main(): rocket = RocketChat(cfg.rocket['user'], cfg.rocket['password'], - server_url='https://coa.crapaud-fou.org') - index = 0 + server_url=cfg.rocket['server']) + labels = [None] * 12 messagesByChannel = [] usersByChannel = [] messagesDataTsunamy = { - "global": [0] * 12, - "project": [0] * 12, - "democraty": [0] * 12, - "ecology": [0] * 12, - "technology": [0] * 12, + Tsunami.GLOBAL: [0] * 12, + Tsunami.PROJECT: [0] * 12, + Tsunami.DEMOCRACY: [0] * 12, + Tsunami.ECOLOGY: [0] * 12, + Tsunami.TECHNOLOGY: [0] * 12, } usersGlobal = [] @@ -63,82 +74,70 @@ def main(): "messagesByChannel": messagesByChannel, "usersByChannel": usersByChannel, "messagesByTsunamy": [ - createElement("global", getColor(), messagesDataTsunamy['global']), - createElement("project", getColor(), messagesDataTsunamy['project']), - createElement("democratie", getColor(), messagesDataTsunamy['democraty']), - createElement("ecologie", getColor(), messagesDataTsunamy['ecology']), - createElement("technologie", getColor(), messagesDataTsunamy['technology']) + createElement("global", getColor(), messagesDataTsunamy[Tsunami.GLOBAL]), + createElement("project", getColor(), messagesDataTsunamy[Tsunami.PROJECT]), + createElement("democratie", getColor(), messagesDataTsunamy[Tsunami.DEMOCRACY]), + createElement("ecologie", getColor(), messagesDataTsunamy[Tsunami.ECOLOGY]), + createElement("technologie", getColor(), messagesDataTsunamy[Tsunami.TECHNOLOGY]) ], "usersGlobal": usersGlobal } - usersTest = [None] * 12 + uniqueUserGlobal = [None] * 12 - while True: - channels = rocket.channels_list(offset=index).json() - - for channel in channels['channels']: - dataMess = [] - dataUsers = [] - pprint( channel['name'] ) - begin = date - monthdelta(12) + for channel in getAllChannels(rocket): + dataMess = [] + dataUsers = [] + print( channel['name'] ) + begin = date - monthdelta(12) + end = begin + monthdelta(1) + + tsunamy = getTsunamy(channel) + + for id in range(0, 12): + if uniqueUserGlobal[id] == None: + uniqueUserGlobal[id] = [] + labels[id] = begin.strftime("%b %Y") + print(f"\t{labels[id]}") + begindate = begin.isoformat() + enddate = end.isoformat() + + resultMess = rocket.channels_history(channel['_id'], oldest= begindate, latest=enddate, count= 10000).json() + resultMess = list(filter(lambda mess: 't' not in mess, resultMess['messages'])) + length = len(resultMess) + dataMess.append(length) + + if length > 0: + setTsunamyInfo(tsunamy, messagesDataTsunamy, id, length) + + users = [] + for mess in resultMess: + users.append(mess['u']['_id']) + uniqueUserGlobal[id].append(mess['u']['_id']) + usersDistinct = set(users) + dataUsers.append(len(usersDistinct)) + else: + dataUsers.append(0) + + begin = end end = begin + monthdelta(1) + + color = getColor() + messageByChannel = createElement(channel['name'], color,dataMess) + userByChannel = createElement( channel['name'], color,dataUsers) - tsunamy = getTsunamy(channel) - - for id in range(0, 12): - if usersTest[id] == None: - usersTest[id] = [] - labels[id] = begin.strftime("%b %Y") - begindate = begin.isoformat() - enddate = end.isoformat() - resultMess = rocket.channels_history(channel['_id'], oldest= begindate, latest=enddate, count= 10000).json() - resultMess = list(filter(lambda mess: 't' not in mess, resultMess['messages'])) - lenght = len(resultMess) - dataMess.append(lenght) - - if lenght > 0: - if tsunamy & Tsunami.GLOBAL: - messagesDataTsunamy['global'][id] += lenght - if tsunamy & Tsunami.PROJECT: - messagesDataTsunamy['project'][id] += lenght - if tsunamy & Tsunami.DEMOCRACY: - messagesDataTsunamy['democraty'][id] += lenght - if tsunamy & Tsunami.ECOLOGY: - messagesDataTsunamy['ecology'][id] += lenght - if tsunamy & Tsunami.TECHNOLOGY: - messagesDataTsunamy['technology'][id] += lenght - - users = [] - for mess in resultMess: - users.append(mess['u']['_id']) - usersTest[id].append(mess['u']['_id']) - usersDistinct = set(users) - dataUsers.append(len(usersDistinct)) - else: - dataUsers.append(0) - - begin = end - end = begin + monthdelta(1) - - color = getColor() - messageByChannel = createElement(channel['name'], color,dataMess) - userByChannel = createElement( channel['name'], color,dataUsers) - - messagesByChannel.append(messageByChannel) - usersByChannel.append(userByChannel) - - if channels['count'] + channels['offset'] >= channels['total']: - break - index += channels['count'] + messagesByChannel.append(messageByChannel) + usersByChannel.append(userByChannel) for id in range(0, 12): - usersTest[id] = len(set(usersTest[id])) + uniqueUserGlobal[id] = len(set(uniqueUserGlobal[id])) - userGlobal = createElement( 'global', 'red', usersTest) + userGlobal = createElement( 'global', 'red', uniqueUserGlobal) usersGlobal.append(userGlobal) save(info) + + if __name__ == "__main__": main() \ No newline at end of file