updated config.py files to move all variables to .env files via Python Decouple

Merge branch 'PR146' into develop

database/.gitignore

@@ -0,0 +1,45 @@
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+pythonenv*
+# Logs
+logs
+*.log
+*_build
+*_static
+*_templates
+#################
+## JetBrains
+#################
+.idea
+
+.vscode
+
+# System
+.DS_Store
+
+# VIM swap files
+.*.swp
+
+# VIM temp files
+*~
+
+#################
+## Sublime Text
+#################
+*.sublime-*
+
+############
+## Windows
+############
+
+# Windows image file caches
+Thumbs.db
+
+# Folder config file
+Desktop.ini

myems-aggregation/.env.example

@@ -0,0 +1,43 @@
+# config for myems_system_db
+MYEMS_SYSTEM_DB_HOST=127.0.0.1
+MYEMS_SYSTEM_DB_PORT=3306
+MYEMS_SYSTEM_DB_DATABASE=myems_system_db
+MYEMS_SYSTEM_DB_USER=root
+MYEMS_SYSTEM_DB_PASSWORD=!MyEMS1
+
+# config for myems_historical_db
+MYEMS_HISTORICAL_DB_HOST=127.0.0.1
+MYEMS_HISTORICAL_DB_PORT=3306
+MYEMS_HISTORICAL_DB_DATABASE=myems_historical_db
+MYEMS_HISTORICAL_DB_USER=root
+MYEMS_HISTORICAL_DB_PASSWORD=!MyEMS1
+
+# config for myems_energy_db
+MYEMS_ENERGY_DB_HOST=127.0.0.1
+MYEMS_ENERGY_DB_PORT=3306
+MYEMS_ENERGY_DB_DATABASE=myems_energy_db
+MYEMS_ENERGY_DB_USER=root
+MYEMS_ENERGY_DB_PASSWORD=!MyEMS1
+
+# config for myems_billing_db
+MYEMS_BILLING_DB_HOST=127.0.0.1
+MYEMS_BILLING_DB_PORT=3306
+MYEMS_BILLING_DB_DATABASE=myems_billing_db
+MYEMS_BILLING_DB_USER=root
+MYEMS_BILLING_DB_PASSWORD=!MyEMS1
+
+# indicates how long in minutes energy data will be aggregated
+# 30 for half hourly
+# 60 for hourly
+MINUTES_TO_COUNT=60
+
+# indicates from when (in UTC timezone) to recalculate if the energy data is null or were cleared
+# format string: "%Y-%m-%d %H:%M:%S"
+START_DATETIME_UTC="2019-12-31 16:00:00"
+
+# indicates the project's time zone offset from UTC
+UTC_OFFSET=+08:00
+
+# the number of worker processes in parallel
+# the pool size depends on the computing performance of the database server and the analysis server
+POOL_SIZE=5

myems-aggregation/README.md

@@ -6,7 +6,17 @@ This service is a component of MyEMS and it aggregates normalized data up to mul
 
 ### Prerequisites
 
-mysql.connector
+mysql-connector-python
+
+python-decouple
+
+### Quick Run for Development
+
+```bash
+pip install -r requirements.txt
+chmod +x run.sh
+run.sh
+```
 
 ### Installation
 
@@ -16,7 +26,7 @@ cd ~/tools
 wget https://dev.mysql.com/get/Downloads/Connector-Python/mysql-connector-python-8.0.20.tar.gz
 tar xzf mysql-connector-python-8.0.20.tar.gz
 cd ~/tools/mysql-connector-python-8.0.20
-sudo python3 setup.py install
+python3 setup.py install
 ```
 
 Install myems-aggregation service:
@@ -24,28 +34,29 @@ Install myems-aggregation service:
 cd ~
 git clone https://github.com/MyEMS/myems.git
 cd myems
-sudo git checkout master (or the latest release tag)
+git checkout master (or the latest release tag)
-sudo cp -R ~/myems/myems-aggregation /myems-aggregation
+cp -R ~/myems/myems-aggregation /myems-aggregation
 ```
-Edit config.py
+Create .env file based on .env.example and edit the .env file if needed:
 ```bash
-sudo nano /myems-aggregation/config.py
+cp /myems-aggregation/.env.example /myems-aggregation/.env
+nano /myems-aggregation/.env
 ```
 Setup systemd service:
 ```bash
-sudo cp myems-aggregation.service /lib/systemd/system/
+cp myems-aggregation.service /lib/systemd/system/
 ```
 Enable the service:
 ```bash
-sudo systemctl enable myems-aggregation.service
+systemctl enable myems-aggregation.service
 ```
 Start the service:
 ```bash
-sudo systemctl start myems-aggregation.service
+systemctl start myems-aggregation.service
 ```
 Monitor the service:
 ```bash
-sudo systemctl status myems-aggregation.service
+systemctl status myems-aggregation.service
 ```
 View the log:
 ```bash

myems-aggregation/config.py

@@ -1,45 +1,79 @@
+from decouple import config
+
+MYEMS_SYSTEM_DB_HOST = config('MYEMS_SYSTEM_DB_HOST', default='127.0.0.1')
+MYEMS_SYSTEM_DB_PORT = config('MYEMS_SYSTEM_DB_PORT', default=3306, cast=int)
+MYEMS_SYSTEM_DB_DATABASE = config('MYEMS_SYSTEM_DB_DATABASE', default='myems_system_db')
+MYEMS_SYSTEM_DB_USER = config('MYEMS_SYSTEM_DB_USER', default='root')
+MYEMS_SYSTEM_DB_PASSWORD = config('MYEMS_SYSTEM_DB_PASSWORD', default='!MyEMS1')
+
+MYEMS_HISTORICAL_DB_HOST = config('MYEMS_HISTORICAL_DB_HOST', default='127.0.0.1')
+MYEMS_HISTORICAL_DB_PORT = config('MYEMS_HISTORICAL_DB_PORT', default=3306, cast=int)
+MYEMS_HISTORICAL_DB_DATABASE = config('MYEMS_HISTORICAL_DB_DATABASE', default='myems_system_db')
+MYEMS_HISTORICAL_DB_USER = config('MYEMS_HISTORICAL_DB_USER', default='root')
+MYEMS_HISTORICAL_DB_PASSWORD = config('MYEMS_HISTORICAL_DB_PASSWORD', default='!MyEMS1')
+
+MYEMS_ENERGY_DB_HOST = config('MYEMS_ENERGY_DB_HOST', default='127.0.0.1')
+MYEMS_ENERGY_DB_PORT = config('MYEMS_ENERGY_DB_PORT', default=3306, cast=int)
+MYEMS_ENERGY_DB_DATABASE = config('MYEMS_ENERGY_DB_DATABASE', default='myems_system_db')
+MYEMS_ENERGY_DB_USER = config('MYEMS_ENERGY_DB_USER', default='root')
+MYEMS_ENERGY_DB_PASSWORD = config('MYEMS_ENERGY_DB_PASSWORD', default='!MyEMS1')
+
+MYEMS_BILLING_DB_HOST = config('MYEMS_BILLING_DB_HOST', default='127.0.0.1')
+MYEMS_BILLING_DB_PORT = config('MYEMS_BILLING_DB_PORT', default=3306, cast=int)
+MYEMS_BILLING_DB_DATABASE = config('MYEMS_BILLING_DB_DATABASE', default='myems_system_db')
+MYEMS_BILLING_DB_USER = config('MYEMS_BILLING_DB_USER', default='root')
+MYEMS_BILLING_DB_PASSWORD = config('MYEMS_BILLING_DB_PASSWORD', default='!MyEMS1')
+
+MINUTES_TO_COUNT = config('MINUTES_TO_COUNT', default=60, cast=int)
+START_DATETIME_UTC = config('START_DATETIME_UTC', default='2019-12-31 16:00:00')
+UTC_OFFSET = config('UTC_OFFSET', default='+08:00')
+POOL_SIZE = config('POOL_SIZE', default=5, cast=int)
+
 myems_system_db = {
-    'user': 'root',
+    'host': MYEMS_SYSTEM_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_SYSTEM_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_SYSTEM_DB_DATABASE,
-    'database': 'myems_system_db',
+    'user': MYEMS_SYSTEM_DB_USER,
+    'password': MYEMS_SYSTEM_DB_PASSWORD,
 }
 
 myems_historical_db = {
-    'user': 'root',
+    'host': MYEMS_HISTORICAL_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_HISTORICAL_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_HISTORICAL_DB_DATABASE,
-    'database': 'myems_historical_db',
+    'user': MYEMS_HISTORICAL_DB_USER,
+    'password': MYEMS_HISTORICAL_DB_PASSWORD,
 }
 
 myems_energy_db = {
-    'user': 'root',
+    'host': MYEMS_ENERGY_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_ENERGY_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_ENERGY_DB_DATABASE,
-    'database': 'myems_energy_db',
+    'user': MYEMS_ENERGY_DB_USER,
+    'password': MYEMS_ENERGY_DB_PASSWORD,
 }
 
 myems_billing_db = {
-    'user': 'root',
+    'host': MYEMS_BILLING_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_BILLING_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_BILLING_DB_DATABASE,
-    'database': 'myems_billing_db',
+    'user': MYEMS_BILLING_DB_USER,
+    'password': MYEMS_BILLING_DB_PASSWORD,
 }
 
-
 # indicates how long in minutes energy data will be aggregated
 # 30 for half hourly
 # 60 for hourly
-minutes_to_count = 60
+minutes_to_count = MINUTES_TO_COUNT
 
 # indicates from when (in UTC timezone) to recalculate if the energy data is null or were cleared
 # format string: '%Y-%m-%d %H:%M:%S'
-start_datetime_utc = '2019-12-31 16:00:00'
+start_datetime_utc = START_DATETIME_UTC
 
 # indicates the project's time zone offset from UTC
-utc_offset = '+08:00'
+utc_offset = UTC_OFFSET
 
 # the number of worker processes in parallel
 # the pool size depends on the computing performance of the database server and the analysis server
-pool_size = 5
+pool_size = POOL_SIZE
 

myems-aggregation/requirements.txt

@@ -1 +1,2 @@
-mysql-connector
+mysql-connector-python
+python-decouple

myems-aggregation/run.sh

@@ -0,0 +1 @@
+python3 main.py

myems-api/.env.example

@@ -0,0 +1,82 @@
+# config for myems_system_db
+MYEMS_SYSTEM_DB_HOST=127.0.0.1
+MYEMS_SYSTEM_DB_PORT=3306
+MYEMS_SYSTEM_DB_DATABASE=myems_system_db
+MYEMS_SYSTEM_DB_USER=root
+MYEMS_SYSTEM_DB_PASSWORD=!MyEMS1
+
+# config for myems_energy_db
+MYEMS_ENERGY_DB_HOST=127.0.0.1
+MYEMS_ENERGY_DB_PORT=3306
+MYEMS_ENERGY_DB_DATABASE=myems_energy_db
+MYEMS_ENERGY_DB_USER=root
+MYEMS_ENERGY_DB_PASSWORD=!MyEMS1
+
+# config for myems_energy_baseline_db
+MYEMS_ENERGY_BASELINE_DB_HOST=127.0.0.1
+MYEMS_ENERGY_BASELINE_DB_PORT=3306
+MYEMS_ENERGY_BASELINE_DB_DATABASE=myems_energy_baseline_db
+MYEMS_ENERGY_BASELINE_DB_USER=root
+MYEMS_ENERGY_BASELINE_DB_PASSWORD=!MyEMS1
+
+# config for myems_billing_db
+MYEMS_BILLING_DB_HOST=127.0.0.1
+MYEMS_BILLING_DB_PORT=3306
+MYEMS_BILLING_DB_DATABASE=myems_billing_db
+MYEMS_BILLING_DB_USER=root
+MYEMS_BILLING_DB_PASSWORD=!MyEMS1
+
+# config for myems_billing_baseline_db
+MYEMS_BILLING_BASELINE_DB_HOST=127.0.0.1
+MYEMS_BILLING_BASELINE_DB_PORT=3306
+MYEMS_BILLING_BASELINE_DB_DATABASE=myems_billing_baseline_db
+MYEMS_BILLING_BASELINE_DB_USER=root
+MYEMS_BILLING_BASELINE_DB_PASSWORD=!MyEMS1
+
+# config for myems_historical_db
+MYEMS_HISTORICAL_DB_HOST=127.0.0.1
+MYEMS_HISTORICAL_DB_PORT=3306
+MYEMS_HISTORICAL_DB_DATABASE=myems_historical_db
+MYEMS_HISTORICAL_DB_USER=root
+MYEMS_HISTORICAL_DB_PASSWORD=!MyEMS1
+
+# config for myems_user_db
+MYEMS_USER_DB_HOST=127.0.0.1
+MYEMS_USER_DB_PORT=3306
+MYEMS_USER_DB_DATABASE=myems_user_db
+MYEMS_USER_DB_USER=root
+MYEMS_USER_DB_PASSWORD=!MyEMS1
+
+# config for myems_fdd_db
+MYEMS_FDD_DB_HOST=127.0.0.1
+MYEMS_FDD_DB_PORT=3306
+MYEMS_FDD_DB_DATABASE=myems_fdd_db
+MYEMS_FDD_DB_USER=root
+MYEMS_FDD_DB_PASSWORD=!MyEMS1
+
+# config for myems_reporting_db
+MYEMS_REPORTING_DB_HOST=127.0.0.1
+MYEMS_REPORTING_DB_PORT=3306
+MYEMS_REPORTING_DB_DATABASE=myems_reporting_db
+MYEMS_REPORTING_DB_USER=root
+MYEMS_REPORTING_DB_PASSWORD=!MyEMS1
+
+# indicated in how many minutes to calculate meter energy consumption
+# 30 for half hourly period
+# 60 for hourly period
+MINUTES_TO_COUNT=60
+
+# indicates the project's time zone offset from UTC
+UTC_OFFSET=+08:00
+
+# indicates from when ( in local timezone) of the day to calculate working days
+WORKING_DAY_START_TIME_LOCAL=00:00:00
+
+# indicates where user uploaded files will be saved to
+# must use the root folder of myems-admin web application
+# for example if you serve myems-admin at /var/www/html/admin
+# you should set the upload_path as below
+UPLOAD_PATH=/var/www/html/admin/upload/
+
+# main currency unit
+CURRENCY_UNIT=CNY

myems-api/.gitignore

@@ -50,3 +50,13 @@ doc/_build
 *~
 
 advancereport/
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+pythonenv*

myems-api/README.md

@@ -19,6 +19,13 @@ gunicorn
 
 openpyxl
 
+## Quick Run for Development
+
+```bash
+pip install -r requirements.txt
+chmod +x run.sh
+run.sh
+```
 
 ## Installation
 
@@ -31,7 +38,7 @@ For macOS developers, please refer to [Installation on macOS (Chinese)](./instal
 cd ~/tools
 git clone https://github.com/c0fec0de/anytree.git
 cd anytree
-sudo python3 setup.py install 
+python3 setup.py install 
 ```
 
 *   Install simplejson
@@ -39,7 +46,7 @@ sudo python3 setup.py install
 cd ~/tools
 git clone https://github.com/simplejson/simplejson.git
 cd simplejson
-sudo python3 setup.py install 
+python3 setup.py install 
 ```
 
 *   Install MySQL Connector
@@ -48,7 +55,7 @@ sudo python3 setup.py install
  wget https://dev.mysql.com/get/Downloads/Connector-Python/mysql-connector-python-8.0.20.tar.gz
  tar xzf mysql-connector-python-8.0.20.tar.gz
  cd ~/tools/mysql-connector-python-8.0.20
- sudo python3 setup.py install
+ python3 setup.py install
 ```
 
 *   Install Falcon,
@@ -67,15 +74,15 @@ sudo python3 setup.py install
  pip3 download cython falcon falcon-cors falcon-multipart
  export LC_ALL="en_US.UTF-8"
  export LC_CTYPE="en_US.UTF-8"
- sudo dpkg-reconfigure locales
+ dpkg-reconfigure locales
- sudo pip3 install --upgrade --no-index --find-links ~/tools/falcon cython falcon falcon-cors falcon-multipart
+ pip3 install --upgrade --no-index --find-links ~/tools/falcon cython falcon falcon-cors falcon-multipart
 ```
 
 *   Install gunicorn, refer to http://gunicorn.org
 ```bash
  mkdir ~/tools/gunicorn && cd ~/tools/gunicorn
  pip3 download gunicorn
- sudo pip3 install --no-index --find-links ~/tools/gunicorn gunicorn
+ pip3 install --no-index --find-links ~/tools/gunicorn gunicorn
 ```
 
 *   Install openpyxl, refer to https://foss.heptapod.net/openpyxl/openpyxl
@@ -104,12 +111,12 @@ tar xzf openpyxl-3.0.7.tar.gz
 
 ```bash
 cd ~/tools/et_xmlfile-1.1
-sudo python3 setup.py install
+python3 setup.py install
 cd ~/tools/jdcal
-sudo python3 setup.py install
+python3 setup.py install
-sudo pip3 install --no-index --find-links ~/tools/pillow Pillow
+pip3 install --no-index --find-links ~/tools/pillow Pillow
 cd ~/tools/openpyxl-3.0.7
-sudo python3 setup.py install
+python3 setup.py install
 ```
 
 *   Download myems:
@@ -121,43 +128,38 @@ git clone https://github.com/MyEMS/myems.git
 *   Install gunicorn service for myems-api:
 ```bash
 cd ~/myems/myems-api
-sudo cp -R ~/myems/myems-api /myems-api
+cp -R ~/myems/myems-api /myems-api
 ```
-  Change IP address in the config file:
+Create .env file based on .env.example and edit the .env file if needed:
 ```bash
-sudo nano /myems-api/config.py
+cp /myems-api/.env.example /myems-api/.env
+nano /myems-api/.env
 ```
 Change the listening port (default is 8000) in gunicorn.socket:
 ```bash
-sudo nano /myems-api/myems-api.socket
+nano /myems-api/myems-api.socket
 ```
 ```bash
 ListenStream=0.0.0.0:8000
 ```
 ```bash
-sudo ufw allow 8000
+ufw allow 8000
 ```
 Setup systemd configure files:
 ```bash
-  sudo cp /myems-api/myems-api.service /lib/systemd/system/
+cp /myems-api/myems-api.service /lib/systemd/system/
-  sudo cp /myems-api/myems-api.socket /lib/systemd/system/
+cp /myems-api/myems-api.socket /lib/systemd/system/
-  sudo cp /myems-api/myems-api.conf /usr/lib/tmpfiles.d/
+cp /myems-api/myems-api.conf /usr/lib/tmpfiles.d/
 ```
    Next enable the services so that they autostart at boot:
 ```bash
-  sudo systemctl enable myems-api.socket
+  systemctl enable myems-api.socket
-  sudo systemctl enable myems-api.service
+  systemctl enable myems-api.service
 ```
 Start the services :
 ```bash
-  sudo systemctl start myems-api.socket
+systemctl start myems-api.socket
-  sudo systemctl start myems-api.service
+systemctl start myems-api.service
-```
-
-## Run for debugging and testing
-```bash
-cd myems-api
-sudo gunicorn -b 127.0.0.1:8000 app:api
 ```
 
 ## API List

myems-api/config.py

@@ -1,82 +1,154 @@
+from decouple import config
+
+
+MYEMS_SYSTEM_DB_HOST = config('MYEMS_SYSTEM_DB_HOST', default='127.0.0.1')
+MYEMS_SYSTEM_DB_PORT = config('MYEMS_SYSTEM_DB_PORT', default=3306, cast=int)
+MYEMS_SYSTEM_DB_DATABASE = config('MYEMS_SYSTEM_DB_DATABASE', default='myems_system_db')
+MYEMS_SYSTEM_DB_USER = config('MYEMS_SYSTEM_DB_USER', default='root')
+MYEMS_SYSTEM_DB_PASSWORD = config('MYEMS_SYSTEM_DB_PASSWORD', default='!MyEMS1')
+
+MYEMS_ENERGY_DB_HOST = config('MYEMS_ENERGY_DB_HOST', default='127.0.0.1')
+MYEMS_ENERGY_DB_PORT = config('MYEMS_ENERGY_DB_PORT', default=3306, cast=int)
+MYEMS_ENERGY_DB_DATABASE = config('MYEMS_ENERGY_DB_DATABASE', default='myems_energy_db')
+MYEMS_ENERGY_DB_USER = config('MYEMS_ENERGY_DB_USER', default='root')
+MYEMS_ENERGY_DB_PASSWORD = config('MYEMS_ENERGY_DB_PASSWORD', default='!MyEMS1')
+
+MYEMS_ENERGY_BASELINE_DB_HOST = config('MYEMS_ENERGY_BASELINE_DB_HOST', default='127.0.0.1')
+MYEMS_ENERGY_BASELINE_DB_PORT = config('MYEMS_ENERGY_BASELINE_DB_PORT', default=3306, cast=int)
+MYEMS_ENERGY_BASELINE_DB_DATABASE = config('MYEMS_ENERGY_BASELINE_DB_DATABASE', default='myems_energy_baseline_db')
+MYEMS_ENERGY_BASELINE_DB_USER = config('MYEMS_ENERGY_BASELINE_DB_USER', default='root')
+MYEMS_ENERGY_BASELINE_DB_PASSWORD = config('MYEMS_ENERGY_BASELINE_DB_PASSWORD', default='!MyEMS1')
+
+MYEMS_BILLING_DB_HOST = config('MYEMS_BILLING_DB_HOST', default='127.0.0.1')
+MYEMS_BILLING_DB_PORT = config('MYEMS_BILLING_DB_PORT', default=3306, cast=int)
+MYEMS_BILLING_DB_DATABASE = config('MYEMS_BILLING_DB_DATABASE', default='myems_billing_db')
+MYEMS_BILLING_DB_USER = config('MYEMS_BILLING_DB_USER', default='root')
+MYEMS_BILLING_DB_PASSWORD = config('MYEMS_BILLING_DB_PASSWORD', default='!MyEMS1')
+
+MYEMS_BILLING_BASELINE_DB_HOST = config('MYEMS_BILLING_BASELINE_DB_HOST', default='127.0.0.1')
+MYEMS_BILLING_BASELINE_DB_PORT = config('MYEMS_BILLING_BASELINE_DB_PORT', default=3306, cast=int)
+MYEMS_BILLING_BASELINE_DB_DATABASE = config('MYEMS_BILLING_BASELINE_DB_DATABASE', default='myems_billing_baseline_db')
+MYEMS_BILLING_BASELINE_DB_USER = config('MYEMS_BILLING_BASELINE_DB_USER', default='root')
+MYEMS_BILLING_BASELINE_DB_PASSWORD = config('MYEMS_BILLING_BASELINE_DB_PASSWORD', default='!MyEMS1')
+
+MYEMS_HISTORICAL_DB_HOST = config('MYEMS_HISTORICAL_DB_HOST', default='127.0.0.1')
+MYEMS_HISTORICAL_DB_PORT = config('MYEMS_HISTORICAL_DB_PORT', default=3306, cast=int)
+MYEMS_HISTORICAL_DB_DATABASE = config('MYEMS_HISTORICAL_DB_DATABASE', default='myems_historical_db')
+MYEMS_HISTORICAL_DB_USER = config('MYEMS_HISTORICAL_DB_USER', default='root')
+MYEMS_HISTORICAL_DB_PASSWORD = config('MYEMS_HISTORICAL_DB_PASSWORD', default='!MyEMS1')
+
+MYEMS_USER_DB_HOST = config('MYEMS_USER_DB_HOST', default='127.0.0.1')
+MYEMS_USER_DB_PORT = config('MYEMS_USER_DB_PORT', default=3306, cast=int)
+MYEMS_USER_DB_DATABASE = config('MYEMS_USER_DB_DATABASE', default='myems_user_db')
+MYEMS_USER_DB_USER = config('MYEMS_USER_DB_USER', default='root')
+MYEMS_USER_DB_PASSWORD = config('MYEMS_USER_DB_PASSWORD', default='!MyEMS1')
+
+MYEMS_FDD_DB_HOST = config('MYEMS_FDD_DB_HOST', default='127.0.0.1')
+MYEMS_FDD_DB_PORT = config('MYEMS_FDD_DB_PORT', default=3306, cast=int)
+MYEMS_FDD_DB_DATABASE = config('MYEMS_FDD_DB_DATABASE', default='myems_fdd_db')
+MYEMS_FDD_DB_USER = config('MYEMS_FDD_DB_USER', default='root')
+MYEMS_FDD_DB_PASSWORD = config('MYEMS_FDD_DB_PASSWORD', default='!MyEMS1')
+
+MYEMS_REPORTING_DB_HOST = config('MYEMS_REPORTING_DB_HOST', default='127.0.0.1')
+MYEMS_REPORTING_DB_PORT = config('MYEMS_REPORTING_DB_PORT', default=3306, cast=int)
+MYEMS_REPORTING_DB_DATABASE = config('MYEMS_REPORTING_DB_DATABASE', default='myems_reporting_db')
+MYEMS_REPORTING_DB_USER = config('MYEMS_REPORTING_DB_USER', default='root')
+MYEMS_REPORTING_DB_PASSWORD = config('MYEMS_REPORTING_DB_PASSWORD', default='!MyEMS1')
+
+MINUTES_TO_COUNT = config('MINUTES_TO_COUNT', default=60, cast=int)
+UTC_OFFSET = config('UTC_OFFSET', default='+08:00')
+WORKING_DAY_START_TIME_LOCAL = config('WORKING_DAY_START_TIME_LOCAL', default='00:00:00')
+UPLOAD_PATH = config('UPLOAD_PATH', default='/var/www/html/admin/upload/')
+CURRENCY_UNIT = config('CURRENCY_UNIT', default='CNY')
+
 myems_system_db = {
-    'user': 'root',
+    'host': MYEMS_SYSTEM_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_SYSTEM_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_SYSTEM_DB_DATABASE,
-    'database': 'myems_system_db',
+    'user': MYEMS_SYSTEM_DB_USER,
+    'password': MYEMS_SYSTEM_DB_PASSWORD,
 }
 
 myems_energy_db = {
-    'user': 'root',
+    'host': MYEMS_ENERGY_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_ENERGY_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_ENERGY_DB_DATABASE,
-    'database': 'myems_energy_db',
+    'user': MYEMS_ENERGY_DB_USER,
+    'password': MYEMS_ENERGY_DB_PASSWORD,
 }
 
 myems_energy_baseline_db = {
-    'user': 'root',
+    'host': MYEMS_ENERGY_BASELINE_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_ENERGY_BASELINE_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_ENERGY_BASELINE_DB_DATABASE,
-    'database': 'myems_energy_baseline_db',
+    'user': MYEMS_ENERGY_BASELINE_DB_USER,
+    'password': MYEMS_ENERGY_BASELINE_DB_PASSWORD,
 }
 
 myems_billing_db = {
-    'user': 'root',
+    'host': MYEMS_BILLING_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_BILLING_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_BILLING_DB_DATABASE,
-    'database': 'myems_billing_db',
+    'user': MYEMS_BILLING_DB_USER,
+    'password': MYEMS_BILLING_DB_PASSWORD,
 }
 
 myems_billing_baseline_db = {
-    'user': 'root',
+    'host': MYEMS_BILLING_BASELINE_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_BILLING_BASELINE_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_BILLING_BASELINE_DB_DATABASE,
-    'database': 'myems_billing_baseline_db',
+    'user': MYEMS_BILLING_BASELINE_DB_USER,
+    'password': MYEMS_BILLING_BASELINE_DB_PASSWORD,
 }
 
 myems_historical_db = {
-    'user': 'root',
+    'host': MYEMS_HISTORICAL_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_HISTORICAL_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_HISTORICAL_DB_DATABASE,
-    'database': 'myems_historical_db',
+    'user': MYEMS_HISTORICAL_DB_USER,
+    'password': MYEMS_HISTORICAL_DB_PASSWORD,
 }
 
 myems_user_db = {
-    'user': 'root',
+    'host': MYEMS_USER_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_USER_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_USER_DB_DATABASE,
-    'database': 'myems_user_db',
+    'user': MYEMS_USER_DB_USER,
+    'password': MYEMS_USER_DB_PASSWORD,
 }
 
 myems_fdd_db = {
-    'user': 'root',
+    'host': MYEMS_FDD_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_FDD_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_FDD_DB_DATABASE,
-    'database': 'myems_fdd_db',
+    'user': MYEMS_FDD_DB_USER,
+    'password': MYEMS_FDD_DB_PASSWORD,
 }
 
 myems_reporting_db = {
-    'user': 'root',
+    'host': MYEMS_REPORTING_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_REPORTING_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_REPORTING_DB_DATABASE,
-    'database': 'myems_reporting_db',
+    'user': MYEMS_REPORTING_DB_USER,
+    'password': MYEMS_REPORTING_DB_PASSWORD,
 }
 
 # indicated in how many minutes to calculate meter energy consumption
 # 30 for half hourly period
 # 60 for hourly period
-minutes_to_count = 60
+minutes_to_count = MINUTES_TO_COUNT
 
 # indicates the project's time zone offset from UTC
-utc_offset = '+08:00'
+utc_offset = UTC_OFFSET
 
 # indicates from when ( in local timezone) of the day to calculate working days
-working_day_start_time_local = '00:00:00'
+working_day_start_time_local = WORKING_DAY_START_TIME_LOCAL
 
 # indicates where user uploaded files will be saved to
 # must use the root folder of myems-admin web application
 # for example if you serve myems-admin at /var/www/html/admin
 # you should set the upload_path as below
-upload_path = '/var/www/html/admin/upload/'
+upload_path = UPLOAD_PATH
 
 # main currency unit
-currency_unit = 'CNY'
+currency_unit = CURRENCY_UNIT

myems-api/requirements.txt

@@ -1,9 +1,12 @@
 anytree
 simplejson
-mysql.connector
+mysql-connector-python
 falcon
 falcon_cors
 falcon-multipart
 gunicorn
+et_xmlfile
+jdcal
 openpyxl
 pillow
+python-decouple

myems-api/run.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+gunicorn --pid pid --timeout 600 --workers=4 app:api

myems-cleaning/.env.example

@@ -0,0 +1,21 @@
+# config for myems_system_db
+MYEMS_SYSTEM_DB_HOST=127.0.0.1
+MYEMS_SYSTEM_DB_PORT=3306
+MYEMS_SYSTEM_DB_DATABASE=myems_system_db
+MYEMS_SYSTEM_DB_USER=root
+MYEMS_SYSTEM_DB_PASSWORD=!MyEMS1
+
+# config for myems_historical_db
+MYEMS_HISTORICAL_DB_HOST=127.0.0.1
+MYEMS_HISTORICAL_DB_PORT=3306
+MYEMS_HISTORICAL_DB_DATABASE=myems_historical_db
+MYEMS_HISTORICAL_DB_USER=root
+MYEMS_HISTORICAL_DB_PASSWORD=!MyEMS1
+
+# indicates how long analog values and digital values will be kept in database
+# the longer days the more memory and disc space needed.
+# NOTE: By default, energy values in historical db will never be deleted automatically.
+LIVE_IN_DAYS=365
+
+# indicates if the program is in debug mode
+IS_DEBUG=False

myems-cleaning/README.md

@@ -9,7 +9,18 @@ This service is a component of MyEMS and it cleans the historical data.
 
 ### Prerequisites
 
-mysql.connector
+mysql-connector-python
+
+schedule
+
+python-decouple
+
+### Quick Run for Development
+```bash
+pip install -r requirements.txt
+chmod +x run.sh
+run.sh
+```
 
 ### Installation
     
@@ -19,7 +30,7 @@ cd ~/tools
 wget https://dev.mysql.com/get/Downloads/Connector-Python/mysql-connector-python-8.0.20.tar.gz
 tar xzf mysql-connector-python-8.0.20.tar.gz
 cd ~/tools/mysql-connector-python-8.0.20
-sudo python3 setup.py install
+python3 setup.py install
 ```
 
 Install myems-cleaning service
@@ -27,28 +38,29 @@ Install myems-cleaning service
 cd ~
 git clone https://github.com/MyEMS/myems.git
 cd myems
-sudo git checkout master (or the latest release tag)
+git checkout master (or the latest release tag)
-sudo cp -R ~/myems/myems-cleaning /myems-cleaning
+cp -R ~/myems/myems-cleaning /myems-cleaning
 ```
-Open config file and edit database configuration
+Create .env file based on .env.example and edit the .env file if needed:
 ```bash
-sudo nano /myems-cleaning/config.py
+cp /myems-cleaning/.env.example /myems-cleaning/.env
+nano /myems-cleaning/.env
 ```
 Setup systemd service:
 ```bash
-sudo cp myems-cleaning.service /lib/systemd/system/
+cp myems-cleaning.service /lib/systemd/system/
 ```
 Enable the service:
 ```bash
-sudo systemctl enable myems-cleaning.service
+systemctl enable myems-cleaning.service
 ```
 Start the service:
 ```bash
-sudo systemctl start myems-cleaning.service
+systemctl start myems-cleaning.service
 ```
 Monitor the service:
 ```bash
-sudo systemctl status myems-cleaning.service
+systemctl status myems-cleaning.service
 ```
 View the log:
 ```bash

myems-cleaning/config.py

@@ -1,24 +1,41 @@
+from decouple import config
+
+
+MYEMS_SYSTEM_DB_HOST = config('MYEMS_SYSTEM_DB_HOST', default='127.0.0.1')
+MYEMS_SYSTEM_DB_PORT = config('MYEMS_SYSTEM_DB_PORT', default=3306, cast=int)
+MYEMS_SYSTEM_DB_DATABASE = config('MYEMS_SYSTEM_DB_DATABASE', default='myems_system_db')
+MYEMS_SYSTEM_DB_USER = config('MYEMS_SYSTEM_DB_USER', default='root')
+MYEMS_SYSTEM_DB_PASSWORD = config('MYEMS_SYSTEM_DB_PASSWORD', default='!MyEMS1')
+
+MYEMS_HISTORICAL_DB_HOST = config('MYEMS_HISTORICAL_DB_HOST', default='127.0.0.1')
+MYEMS_HISTORICAL_DB_PORT = config('MYEMS_HISTORICAL_DB_PORT', default=3306, cast=int)
+MYEMS_HISTORICAL_DB_DATABASE = config('MYEMS_HISTORICAL_DB_DATABASE', default='myems_historical_db')
+MYEMS_HISTORICAL_DB_USER = config('MYEMS_HISTORICAL_DB_USER', default='root')
+MYEMS_HISTORICAL_DB_PASSWORD = config('MYEMS_HISTORICAL_DB_PASSWORD', default='!MyEMS1')
+
+LIVE_IN_DAYS = config('LIVE_IN_DAYS', default=365, cast=int)
+IS_DEBUG = config('IS_DEBUG', default=False, cast=bool)
+
 myems_system_db = {
-    'user': 'root',
+    'host': MYEMS_SYSTEM_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_SYSTEM_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_SYSTEM_DB_DATABASE,
-    'database': 'myems_system_db',
+    'user': MYEMS_SYSTEM_DB_USER,
-    'port': 3306,
+    'password': MYEMS_SYSTEM_DB_PASSWORD,
 }
 
 myems_historical_db = {
-    'user': 'root',
+    'host': MYEMS_HISTORICAL_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_HISTORICAL_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_HISTORICAL_DB_DATABASE,
-    'database': 'myems_historical_db',
+    'user': MYEMS_HISTORICAL_DB_USER,
-    'port': 3306,
+    'password': MYEMS_HISTORICAL_DB_PASSWORD,
 }
 
 # indicates how long analog values and digital values will be kept in database
 # the longer days the more memory and disc space needed.
-live_in_days = 365
-
 # NOTE: By default, energy values in historical db will never be deleted automatically.
+live_in_days = LIVE_IN_DAYS
 
 # indicates if the program is in debug mode
-is_debug = False
+is_debug = IS_DEBUG

myems-cleaning/requirements.txt

@@ -1,2 +1,3 @@
-mysql-connector
+mysql-connector-python
 schedule
+python-decouple

myems-cleaning/run.sh

@@ -0,0 +1 @@
+python main.py

myems-modbus-tcp/.env.example

@@ -0,0 +1,21 @@
+# config for myems_system_db
+MYEMS_SYSTEM_DB_HOST=127.0.0.1
+MYEMS_SYSTEM_DB_PORT=3306
+MYEMS_SYSTEM_DB_DATABASE=myems_system_db
+MYEMS_SYSTEM_DB_USER=root
+MYEMS_SYSTEM_DB_PASSWORD=!MyEMS1
+
+# config for myems_historical_db
+MYEMS_HISTORICAL_DB_HOST=127.0.0.1
+MYEMS_HISTORICAL_DB_PORT=3306
+MYEMS_HISTORICAL_DB_DATABASE=myems_historical_db
+MYEMS_HISTORICAL_DB_USER=root
+MYEMS_HISTORICAL_DB_PASSWORD=!MyEMS1
+
+# Indicates how long the process waits between readings
+INTERVAL_IN_SECONDS=600
+
+# Get the gateway ID and token from MyEMS Admin
+# This is used for getting data sources associated with the gateway
+GATEWAY_ID=1
+GATEWAY_TOKEN=AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA

myems-modbus-tcp/README.md

@@ -4,13 +4,23 @@
 This service is a component of MyEMS to acquire data from Modbus TCP devices.
 
 ### Prerequisites
-pyserial
 
-modbus-tk
+mysql-connector-python
 
-mysql.connector
+modbus_tk
 
-Schedule
+schedule
+
+python-decouple
+
+
+### Quick Run for Development
+
+```bash
+pip install -r requirements.txt
+chmod +x run.sh
+run.sh
+```
 
 ### Installation
 
@@ -50,9 +60,10 @@ cd myems
 git checkout master (or the latest release tag)
 cp -R ~/myems/myems-modbus-tcp /myems-modbus-tcp
 ```
-Edit the config
+Create .env file based on .env.example and edit the .env file if needed:
 ```bash
-nano /myems-modbus-tcp/config.py
+cp /myems-modbus-tcp/.env.example /myems-cleaning/.env
+nano /myems-modbus-tcp/.env
 ```
 Setup systemd service:
 ```bash

myems-modbus-tcp/config.py

@@ -1,25 +1,45 @@
+from decouple import config
+
+
+MYEMS_SYSTEM_DB_HOST = config('MYEMS_SYSTEM_DB_HOST', default='127.0.0.1')
+MYEMS_SYSTEM_DB_PORT = config('MYEMS_SYSTEM_DB_PORT', default=3306, cast=int)
+MYEMS_SYSTEM_DB_DATABASE = config('MYEMS_SYSTEM_DB_DATABASE', default='myems_system_db')
+MYEMS_SYSTEM_DB_USER = config('MYEMS_SYSTEM_DB_USER', default='root')
+MYEMS_SYSTEM_DB_PASSWORD = config('MYEMS_SYSTEM_DB_PASSWORD', default='!MyEMS1')
+
+MYEMS_HISTORICAL_DB_HOST = config('MYEMS_HISTORICAL_DB_HOST', default='127.0.0.1')
+MYEMS_HISTORICAL_DB_PORT = config('MYEMS_HISTORICAL_DB_PORT', default=3306, cast=int)
+MYEMS_HISTORICAL_DB_DATABASE = config('MYEMS_HISTORICAL_DB_DATABASE', default='myems_historical_db')
+MYEMS_HISTORICAL_DB_USER = config('MYEMS_HISTORICAL_DB_USER', default='root')
+MYEMS_HISTORICAL_DB_PASSWORD = config('MYEMS_HISTORICAL_DB_PASSWORD', default='!MyEMS1')
+
+INTERVAL_IN_SECONDS = config('INTERVAL_IN_SECONDS', default=600, cast=int)
+GATEWAY_ID = config('GATEWAY_ID', cast=int)
+GATEWAY_TOKEN = config('GATEWAY_TOKEN')
+
 myems_system_db = {
-    'user': 'root',
+    'host': MYEMS_SYSTEM_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_SYSTEM_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_SYSTEM_DB_DATABASE,
-    'database': 'myems_system_db',
+    'user': MYEMS_SYSTEM_DB_USER,
-    'port': 3306,
+    'password': MYEMS_SYSTEM_DB_PASSWORD,
 }
 
 myems_historical_db = {
-    'user': 'root',
+    'host': MYEMS_HISTORICAL_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_HISTORICAL_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_HISTORICAL_DB_DATABASE,
-    'database': 'myems_historical_db',
+    'user': MYEMS_HISTORICAL_DB_USER,
-    'port': 3306,
+    'password': MYEMS_HISTORICAL_DB_PASSWORD,
 }
 
+
 # Indicates how long the process waits between readings
-interval_in_seconds = 600
+interval_in_seconds = INTERVAL_IN_SECONDS
 
 # Get the gateway ID and token from MyEMS Admin
 # This is used for getting data sources associated with the gateway
 gateway = {
-    'id': 1,
+    'id': GATEWAY_ID,
-    'token': 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'
+    'token': GATEWAY_TOKEN
 }

myems-modbus-tcp/requirements.txt

@@ -1,3 +1,4 @@
-mysql-connector
+mysql-connector-python
 modbus_tk
 schedule
+python-decouple

myems-modbus-tcp/run.sh

@@ -0,0 +1 @@
+python main.py

myems-normalization/.env.example

@@ -0,0 +1,46 @@
+# config for myems_system_db
+MYEMS_SYSTEM_DB_HOST=127.0.0.1
+MYEMS_SYSTEM_DB_PORT=3306
+MYEMS_SYSTEM_DB_DATABASE=myems_system_db
+MYEMS_SYSTEM_DB_USER=root
+MYEMS_SYSTEM_DB_PASSWORD=!MyEMS1
+
+# config for myems_historical_db
+MYEMS_HISTORICAL_DB_HOST=127.0.0.1
+MYEMS_HISTORICAL_DB_PORT=3306
+MYEMS_HISTORICAL_DB_DATABASE=myems_historical_db
+MYEMS_HISTORICAL_DB_USER=root
+MYEMS_HISTORICAL_DB_PASSWORD=!MyEMS1
+
+# config for myems_energy_db
+MYEMS_ENERGY_DB_HOST=127.0.0.1
+MYEMS_ENERGY_DB_PORT=3306
+MYEMS_ENERGY_DB_DATABASE=myems_energy_db
+MYEMS_ENERGY_DB_USER=root
+MYEMS_ENERGY_DB_PASSWORD=!MyEMS1
+
+# config for myems_billing_db
+MYEMS_BILLING_DB_HOST=127.0.0.1
+MYEMS_BILLING_DB_PORT=3306
+MYEMS_BILLING_DB_DATABASE=myems_billing_db
+MYEMS_BILLING_DB_USER=root
+MYEMS_BILLING_DB_PASSWORD=!MyEMS1
+
+# indicates in how many minutes to normalize energy consumption
+# 30 for half hourly
+# 60 for hourly
+MINUTES_TO_COUNT=60
+
+# indicates within how many minutes to allow myems-cleaning service to clean the historical data
+MINUTES_TO_CLEAN=30
+
+# indicates from when (in UTC timezone) to calculate if the energy data is empty or were cleared
+# format string: "%Y-%m-%d %H:%M:%S"
+START_DATETIME_UTC="2019-12-31 16:00:00"
+
+# indicates the project's time zone offset from UTC
+UTC_OFFSET=+08:00
+
+# the number of worker processes in parallel for meter and virtual meter
+# the pool size depends on the computing performance of the database server and the analysis server
+POOL_SIZE=5

myems-normalization/README.md

@@ -8,11 +8,22 @@ This service is a component of MyEMS and it normalizes energy data in historical
 
 ### Prerequisites
 
-mysql.connector
+mysql-connector-python
+
+openpyxl
 
 sympy
 
-openpyxl
+python-decouple
+
+
+### Quick Run for Development
+
+```bash
+pip install -r requirements.txt
+chmod +x run.sh
+run.sh
+```
 
 ### Installation
 
@@ -22,7 +33,7 @@ cd ~/tools
 wget https://dev.mysql.com/get/Downloads/Connector-Python/mysql-connector-python-8.0.20.tar.gz
 tar xzf mysql-connector-python-8.0.20.tar.gz
 cd ~/tools/mysql-connector-python-8.0.20
-sudo python3 setup.py install
+python3 setup.py install
 ```
 
 Download and install mpmath
@@ -30,7 +41,7 @@ Download and install mpmath
 cd ~/tools
 git clone https://github.com/fredrik-johansson/mpmath.git
 cd ~/tools/mpmath
-sudo python3 setup.py install
+python3 setup.py install
 ```
 
 Download and install SymPy
@@ -38,7 +49,7 @@ Download and install SymPy
 cd ~/tools
 git clone https://github.com/sympy/sympy.git
 cd ~/tools/sympy
-sudo python3 setupegg.py develop
+python3 setupegg.py develop
 ```
 
 Download and install openpyxl
@@ -62,11 +73,11 @@ tar xzf openpyxl-3.0.7.tar.gz
 
 ```bash
 cd ~/tools/et_xmlfile-1.1
-sudo python3 setup.py install
+python3 setup.py install
 cd ~/tools/jdcal
-sudo python3 setup.py install
+python3 setup.py install
 cd ~/tools/openpyxl-3.0.7
-sudo python3 setup.py install
+python3 setup.py install
 ```
 
 Install myems-normalization service:
@@ -74,29 +85,29 @@ Install myems-normalization service:
 cd ~
 git clone https://github.com/MyEMS/myems.git
 cd myems
-sudo git checkout master (or the latest release tag)
+git checkout master (or the latest release tag)
-sudo cp -r ~/myems/myems-normalization /myems-normalization
+cp -r ~/myems/myems-normalization /myems-normalization
 ```
-
+Create .env file based on .env.example and edit the .env file if needed:
-Edit config.py
+```bash
-```
+cp /myems-normalization/.env.example /myems-cleaning/.env
-sudo nano /myems-normalization/config.py
+nano /myems-normalization/.env
 ```
 Setup systemd service:
 ```
-sudo cp myems-normalization.service /lib/systemd/system/
+cp myems-normalization.service /lib/systemd/system/
 ```
 Enable the service:
 ```
-sudo systemctl enable myems-normalization.service
+systemctl enable myems-normalization.service
 ```
 Start the service:
 ```
-sudo systemctl start myems-normalization.service
+systemctl start myems-normalization.service
 ```
 Monitor the service:
 ```bash
-sudo systemctl status myems-normalization.service
+systemctl status myems-normalization.service
 ```
 View the log:
 ```bash

myems-normalization/config.py

@@ -1,43 +1,76 @@
+from decouple import config
+
+
+MYEMS_SYSTEM_DB_HOST = config('MYEMS_SYSTEM_DB_HOST', default='127.0.0.1')
+MYEMS_SYSTEM_DB_PORT = config('MYEMS_SYSTEM_DB_PORT', default=3306, cast=int)
+MYEMS_SYSTEM_DB_DATABASE = config('MYEMS_SYSTEM_DB_DATABASE', default='myems_system_db')
+MYEMS_SYSTEM_DB_USER = config('MYEMS_SYSTEM_DB_USER', default='root')
+MYEMS_SYSTEM_DB_PASSWORD = config('MYEMS_SYSTEM_DB_PASSWORD', default='!MyEMS1')
+
+MYEMS_HISTORICAL_DB_HOST = config('MYEMS_HISTORICAL_DB_HOST', default='127.0.0.1')
+MYEMS_HISTORICAL_DB_PORT = config('MYEMS_HISTORICAL_DB_PORT', default=3306, cast=int)
+MYEMS_HISTORICAL_DB_DATABASE = config('MYEMS_HISTORICAL_DB_DATABASE', default='myems_system_db')
+MYEMS_HISTORICAL_DB_USER = config('MYEMS_HISTORICAL_DB_USER', default='root')
+MYEMS_HISTORICAL_DB_PASSWORD = config('MYEMS_HISTORICAL_DB_PASSWORD', default='!MyEMS1')
+
+MYEMS_ENERGY_DB_HOST = config('MYEMS_ENERGY_DB_HOST', default='127.0.0.1')
+MYEMS_ENERGY_DB_PORT = config('MYEMS_ENERGY_DB_PORT', default=3306, cast=int)
+MYEMS_ENERGY_DB_DATABASE = config('MYEMS_ENERGY_DB_DATABASE', default='myems_system_db')
+MYEMS_ENERGY_DB_USER = config('MYEMS_ENERGY_DB_USER', default='root')
+MYEMS_ENERGY_DB_PASSWORD = config('MYEMS_ENERGY_DB_PASSWORD', default='!MyEMS1')
+
+MYEMS_BILLING_DB_HOST = config('MYEMS_BILLING_DB_HOST', default='127.0.0.1')
+MYEMS_BILLING_DB_PORT = config('MYEMS_BILLING_DB_PORT', default=3306, cast=int)
+MYEMS_BILLING_DB_DATABASE = config('MYEMS_BILLING_DB_DATABASE', default='myems_system_db')
+MYEMS_BILLING_DB_USER = config('MYEMS_BILLING_DB_USER', default='root')
+MYEMS_BILLING_DB_PASSWORD = config('MYEMS_BILLING_DB_PASSWORD', default='!MyEMS1')
+
+MINUTES_TO_COUNT = config('MINUTES_TO_COUNT', default=60, cast=int)
+MINUTES_TO_CLEAN = config('MINUTES_TO_CLEAN', default=30, cast=int)
+START_DATETIME_UTC = config('START_DATETIME_UTC', default='2019-12-31 16:00:00')
+UTC_OFFSET = config('UTC_OFFSET', default='+08:00')
+POOL_SIZE = config('POOL_SIZE', default=5, cast=int)
+
 myems_system_db = {
-    'user': 'root',
+    'host': MYEMS_SYSTEM_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_SYSTEM_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_SYSTEM_DB_DATABASE,
-    'database': 'myems_system_db',
+    'user': MYEMS_SYSTEM_DB_USER,
-    'port': 3306,
+    'password': MYEMS_SYSTEM_DB_PASSWORD,
 }
 
 myems_energy_db = {
-    'user': 'root',
+    'host': MYEMS_ENERGY_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_ENERGY_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_ENERGY_DB_DATABASE,
-    'database': 'myems_energy_db',
+    'user': MYEMS_ENERGY_DB_USER,
-    'port': 3306,
+    'password': MYEMS_ENERGY_DB_PASSWORD,
 }
 
 myems_historical_db = {
-    'user': 'root',
+    'host': MYEMS_HISTORICAL_DB_HOST,
-    'password': '!MyEMS1',
+    'port': MYEMS_HISTORICAL_DB_PORT,
-    'host': '127.0.0.1',
+    'database': MYEMS_HISTORICAL_DB_DATABASE,
-    'database': 'myems_historical_db',
+    'user': MYEMS_HISTORICAL_DB_USER,
-    'port': 3306,
+    'password': MYEMS_HISTORICAL_DB_PASSWORD,
 }
 
 # indicates in how many minutes to normalize energy consumption
 # 30 for half hourly
 # 60 for hourly
-minutes_to_count = 60
+minutes_to_count = MINUTES_TO_COUNT
 
 # indicates within how many minutes to allow myems-cleaning service to clean the historical data
-minutes_to_clean = 30
+minutes_to_clean = MINUTES_TO_CLEAN
 
 # indicates from when (in UTC timezone) to calculate if the energy data is empty or were cleared
-# format string: '%Y-%m-%d %H:%M:%S'
+# format string: "%Y-%m-%d %H:%M:%S"
-start_datetime_utc = '2019-12-31 16:00:00'
+start_datetime_utc = START_DATETIME_UTC
+
+# indicates the project's time zone offset from UTC
+utc_offset = UTC_OFFSET
 
 # the number of worker processes in parallel for meter and virtual meter
 # the pool size depends on the computing performance of the database server and the analysis server
-pool_size = 5
+pool_size = POOL_SIZE
-
-# indicates the project's time zone offset from UTC
-utc_offset = '+08:00'
 

myems-normalization/requirements.txt

@@ -1,3 +1,4 @@
-mysql-connector
+mysql-connector-python
 openpyxl
 sympy
+python-decouple

myems-normalization/run.sh

@@ -0,0 +1 @@
+python main.py