Merge branch 'updatestats'

.gitignore

@@ -6,3 +6,5 @@ vendor/
 .bundle
 script/dev.config.py
 script/__pycache__/*.pyc
+script/common/__pycache__/*.pyc
+\.vscode/\.ropeproject/

_includes/head.html

@@ -84,6 +84,9 @@
 <script src="{{ site.baseurl }}/public/js/cytoscape/cytoscape.min.js"></script>
 <script src="{{ site.baseurl }}/public/js/cytoscape/cytoscape-automove.js"></script>
 {% endif %}
+{% if page.chartjs %}
+<script src="{{ site.baseurl }}/public/js/chartjs/Chart.bundle.min.js"></script>
+{% endif %}
 
 <!-- Icons -->
 <!-- 16x16 -->

pages/channelsstats.md

@@ -0,0 +1,131 @@
+---
+layout: page
+permalink: /channelsstats/
+title: Statistiques du Chat
+linktitle: "Statistiques du Chat"
+linkurl: /channelsstats/
+description: "Statistiques sur le chat https://coa.crapaud-fou.org"
+chartjs: true
+jquery: true
+---
+
+Statistiques du <a href="https://coa.crapaud-fou.org">chat</a> (aussi connu sous le nom de mare aux crapauds) :
+<canvas id="byChannel"></canvas>
+<canvas id="byTsunamy"></canvas>
+<canvas id="usersByChannel"></canvas>
+<script>
+
+$.getJSON("{{ site.baseurl }}/public/data/messagesByChannel.json", function (datas){
+    updated = datas['updated']
+
+    labels = datas['labels'];
+
+    var ctx = document.getElementById('byChannel').getContext('2d');
+    var chart = new Chart(ctx, {
+        // The type of chart we want to create
+        type: 'bar',
+
+        // The data for our dataset
+        data: {
+            labels: labels,
+            datasets: datas['messagesByChannel'],
+        },
+
+        // Configuration options go here
+        options: {
+            legend: {
+                display: false
+            },
+            title: {
+                display: true,
+                text: "Nombre de message par canaux (" + updated + ")",
+                position: "top"
+            },
+            responsive: true,
+            scales: {
+                xAxes: [{
+                    stacked: true
+                }],
+                yAxes: [{
+                    stacked: true,
+                    ticks: {
+                        stepSize: 500
+                    }
+                }]
+            }
+        }
+    });
+
+    var ctx2 = document.getElementById('byTsunamy').getContext('2d');
+    var chart2 = new Chart(ctx2, {
+        // The type of chart we want to create
+        type: 'bar',
+
+        // The data for our dataset
+        data: {
+            labels: labels,
+            datasets: datas['messagesByTsunamy'],
+        },
+
+        // Configuration options go here
+        options: {
+            legend: {
+                display: false
+            },
+            title: {
+                display: true,
+                text: "Nombre de message par Tsunami (" + updated + ")",
+                position: "top"
+            },
+            responsive: true,
+            scales: {
+                xAxes: [{
+                    stacked: true
+                }],
+                yAxes: [{
+                    stacked: true,
+                    ticks: {
+                        stepSize: 500
+                    }
+                }]
+            }
+        }
+    });
+
+    var ctx3 = document.getElementById('usersByChannel').getContext('2d');
+    var chart3 = new Chart(ctx3, {
+        // The type of chart we want to create
+        type: 'bar',
+
+        // The data for our dataset
+        data: {
+            labels: labels,
+            datasets: datas['usersByChannel'],
+        },
+
+        // Configuration options go here
+        options: {
+            legend: {
+                display: false
+            },
+            title: {
+                display: true,
+                text: "Nombre d'utilisateur actifs par canaux (" + updated + ")",
+                position: "top"
+            },
+            responsive: true,
+            scales: {
+                xAxes: [{
+                    stacked: true
+                }],
+                yAxes: [{
+                    stacked: true,
+                    ticks: {
+                        stepSize: 50
+                    }
+                }]
+            }
+        }
+    });
+});
+</script>

script/README.md

@@ -1,5 +1,4 @@
-# getchannels
+# installation
-## installation
 ## pre-requis
 Il faut Python 3 & pip car le code est fait en python.
 ## Installation
@@ -13,8 +12,25 @@ Il faut créer un fichier dev_config.py avec ses infos de connection au Rocket.c
         'user' : 'username',
         'password': 'pwd'
     }
-# Lancer
+# Get Channels
+Permet de générer les infos pour la page  
+https://crapaud-fou.org/channelslist/
+## Lancer
     python getchannels.py
 
-# Commit
+## Commit
-Un fichier sera créé dans le répertoire [../public/data/result.json](../public/data/result.json)
+Des fichiers seront créé
+* [../public/data/channelslist.json](../public/data/channelslist.json)
+* [../public/data/cohortescolor.json](../public/data/cohortescolor.json)
+* [../public/data/cohorteslist.json](../public/data/cohorteslist.json)
+
+# Get Stats
+Permet de générer les infos pour la page  
+https://crapaud-fou.org/channelsstats/
+
+## Lancer
+    python getstats.py
+
+## Commit
+Un fichier sera créé
+* [../public/data/channelsstat.json](../public/data/channelsstat.json)

script/common/channelhelper.py

@@ -0,0 +1,39 @@
+def getNodesOrigin(channel):
+  nodes = []
+  if 'description' not in channel:
+    nodes.append("global")
+    return nodes
+
+  if channel['description'].find("#projet") != -1:
+    nodes.append("project")
+  if channel['description'].find("#democratie") != -1:
+    nodes.append("democratie")
+  if channel['description'].find("#ecologie") != -1:
+    nodes.append("ecologie")
+  if channel['description'].find("#technologie") != -1:
+    nodes.append("technologie")
+
+  if not nodes:
+    nodes.append("global")
+
+  return nodes
+
+def getTsunamy(channel):
+    value = Tsunami.GLOBAL
+    if 'description' in channel:
+        if channel['description'].find("#projet") != -1:
+            value |= Tsunami.PROJECT
+        if channel['description'].find("#democratie") != -1:
+            value |= Tsunami.DEMOCRACY
+        if channel['description'].find("#ecologie") != -1:
+            value |= Tsunami.ECOLOGY
+        if channel['description'].find("#technologie") != -1:
+            value |= Tsunami.TECHNOLOGY
+    return value
+
+class Tsunami:
+    GLOBAL      = 1 << 0
+    PROJECT     = 1 << 1
+    DEMOCRACY   = 1 << 2
+    ECOLOGY     = 1 << 3
+    TECHNOLOGY  = 1 << 4

script/getchannels.py

@@ -9,26 +9,7 @@ import json
 import dev_config as cfg
 import os
 import re
-
+from common.channelhelper import getNodesOrigin
-def getNodesOrigin(channel):
-  nodes = []
-  if 'description' not in channel:
-    nodes.append("global")
-    return nodes
-
-  if channel['description'].find("#projet") != -1:
-    nodes.append("project")
-  if channel['description'].find("#democratie") != -1:
-    nodes.append("democratie")
-  if channel['description'].find("#ecologie") != -1:
-    nodes.append("ecologie")
-  if channel['description'].find("#technologie") != -1:
-    nodes.append("technologie")
-
-  if not nodes:
-    nodes.append("global")
-
-  return nodes
 
 colorInfo = { 
   'global': 'orange',
@@ -73,7 +54,7 @@ while True:
   for channel in channels['channels']:
     if channel['name'].find('cohorte') != -1:
       if 'description' in channel:
-        m = re.findall('#([\w-]+)', channel['description'])
+        m = re.findall(r'#([\w-]+)', channel['description'])
         for region in m:
           cohortescolor.update( { region: 'green' } )
           cohortes.update( { region: { 'link': channel['name']}})

script/getstats.py

@@ -0,0 +1,148 @@
+# coding: utf8
+
+# toutes les chaines sont en unicode (même les docstrings)
+from __future__ import unicode_literals
+
+from pprint import pprint
+from rocketchat_API.rocketchat import RocketChat
+import json
+import dev_config as cfg
+import os
+import random
+from datetime import datetime
+from monthdelta import monthdelta
+from common.channelhelper import getTsunamy
+from common.channelhelper import Tsunami
+
+def getColor():
+    r = random.randrange(255)
+    g = random.randrange(255)
+    b = random.randrange(255)
+    return 'rgb({:0},{:0},{:0})'.format(r,g,b)
+
+rocket = RocketChat(cfg.rocket['user'], cfg.rocket['password'],
+                    server_url='https://coa.crapaud-fou.org')
+index = 0
+labels = [None] * 12
+messagesByChannel = []
+messagesByTsunamy = []
+usersByChannel = []
+messagesDataTsunamy = {
+  "global": [0] * 12,
+  "project": [0] * 12,
+  "democraty": [0] * 12,
+  "ecology": [0] * 12,
+  "technology": [0] * 12,
+}
+
+now = datetime.now()
+date = datetime(now.year, now.month, now.day, 0,0,0)
+
+info = {
+  "updated": "updated {:02}/{:02}/{:04}".format(now.day, now.month, now.year),
+  "labels": labels,
+  "messagesByChannel": messagesByChannel,
+  "usersByChannel": usersByChannel,
+  "messagesByTsunamy": [{        
+        "label": "global",
+        "backgroundColor": getColor(),
+        "data": messagesDataTsunamy['global']
+    },
+    {        
+        "label": "projet",
+        "backgroundColor": getColor(),
+        "data": messagesDataTsunamy['project']
+    },
+    {        
+        "label": "democratie",
+        "backgroundColor": getColor(),
+        "data": messagesDataTsunamy['democraty']
+    },
+    {        
+        "label": "ecologie",
+        "backgroundColor": getColor(),
+        "data": messagesDataTsunamy['ecology']
+    },
+    {        
+        "label": "technologie",
+        "backgroundColor": getColor(),
+        "data": messagesDataTsunamy['technology']
+    }]
+}
+
+while True:
+  channels = rocket.channels_list(offset=index).json()
+  totalChannels = channels['total']
+    
+  for channel in channels['channels']:
+    dataMess = []
+    dataUsers = []
+    pprint( channel['name'] )
+    begin = date - monthdelta(12)
+    end = begin + monthdelta(1)
+
+    tsunamy = getTsunamy(channel)    
+
+    for id in range(0, 12):
+      labels[id] = begin.strftime("%b %Y")
+      begindate =  begin.isoformat()
+      enddate = end.isoformat()
+      resultMess = rocket.channels_history(channel['_id'], oldest= begindate, latest=enddate, count= 10000).json()
+      lenght = len(resultMess['messages'])
+      dataMess.append(lenght)
+
+      if lenght > 0:
+        if tsunamy & Tsunami.GLOBAL:
+          messagesDataTsunamy['global'][id] += lenght
+        if tsunamy & Tsunami.PROJECT:
+          messagesDataTsunamy['project'][id] += lenght
+        if tsunamy & Tsunami.DEMOCRACY:
+          messagesDataTsunamy['democraty'][id] += lenght
+        if tsunamy & Tsunami.ECOLOGY:
+          messagesDataTsunamy['ecology'][id] += lenght
+        if tsunamy & Tsunami.TECHNOLOGY:
+          messagesDataTsunamy['technology'][id] += lenght
+
+        users = []
+        for mess in resultMess['messages']:
+          users.append(mess['u']['_id'])      
+        usersDistinct = set(users)
+        dataUsers.append(len(usersDistinct))
+      else:
+        dataUsers.append(0)
+
+      begin = end
+      end = begin + monthdelta(1)
+   
+    color = getColor()
+    messageByChannel = {
+        "label": channel['name'],
+        "backgroundColor": color,
+        "data": dataMess
+    }
+
+    userByChannel = {
+        "label": channel['name'],
+        "backgroundColor": color,
+        "data": dataUsers
+    }
+
+    messagesByChannel.append(messageByChannel)
+    usersByChannel.append(userByChannel)
+
+  if channels['count'] + channels['offset'] >= channels['total']:
+    break
+  index += channels['count']
+
+# Récupération du répertoire racine du repo
+rootFolder = os.path.join(os.path.dirname(__file__), '..')
+# Répertoire pour stocker le fichier de sortie
+dataFolder = os.path.join(rootFolder, 'public', 'data')
+
+
+
+statsFilePath = os.path.abspath(
+    os.path.join(dataFolder, 'channelsstat.json'))
+with open(statsFilePath, "w") as file_write:
+  json.dump(info, file_write)
+

script/requirements.txt

@@ -1 +1,2 @@
 rocketchat_API==0.6.32
+MonthDelta==0.9.1