From 4a8f06af83445240338132a9b021ce48115f22df Mon Sep 17 00:00:00 2001 From: winnerineast Date: Fri, 1 Oct 2021 13:45:30 +0800 Subject: [PATCH 1/7] created batch script to initialize the database. --- database/README.md | 2 +- database/create_database_ubuntu.sh | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 database/create_database_ubuntu.sh diff --git a/database/README.md b/database/README.md index 2a120a8e..a31696e3 100644 --- a/database/README.md +++ b/database/README.md @@ -13,7 +13,7 @@ Providing database schema and scripts for MyEMS. ### Installation -Execute the following scripts in MySQL commandline, or execute in other MySQL client tools such as MySQL Workbench, Navicat, DBaver, phpMyAdmin, etc. +Execute the following scripts in MySQL commandline, or execute in other MySQL client tools such as MySQL Workbench, Navicat, DBaver, phpMyAdmin, etc. Here is [an example shell script](create_database_ubuntu.sh) to initiate database in one command on Ubuntu 20.04. ```bash mysql -u root -p < myems_billing_baseline_db.sql mysql -u root -p < myems_billing_db.sql diff --git a/database/create_database_ubuntu.sh b/database/create_database_ubuntu.sh new file mode 100644 index 00000000..ab4063bb --- /dev/null +++ b/database/create_database_ubuntu.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +mypassword="pAssw0rd@12345" + +mysql -uroot -p$mypassword < myems_billing_baseline_db.sql +mysql -uroot -p$mypassword < myems_billing_db.sql +mysql -uroot -p$mypassword < myems_energy_baseline_db.sql +mysql -uroot -p$mypassword < myems_energy_db.sql +mysql -uroot -p$mypassword < myems_fdd_db.sql +mysql -uroot -p$mypassword < myems_historical_db.sql +mysql -uroot -p$mypassword < myems_reporting_db.sql +mysql -uroot -p$mypassword < myems_system_db.sql +mysql -uroot -p$mypassword < myems_user_db.sql +mysql -uroot -p$mypassword < demo/myems_system_db.sql \ No newline at end of file From 1bfdedf0cc9b29c758d4c842304befbfb08abffa Mon Sep 17 00:00:00 2001 From: winnerineast Date: Fri, 1 Oct 2021 18:38:34 +0800 Subject: [PATCH 2/7] fixed one bug for database. created a dirty and easy way to run myems-api. enhanced requirements.txt for both database and myems-api. --- database/create_database_ubuntu.sh | 2 +- myems-api/README.md | 8 +++++++- myems-api/config.py | 18 +++++++++--------- myems-api/myems-api.service | 8 ++++---- myems-api/requirements.txt | 5 ++++- myems-api/run.sh | 3 +++ 6 files changed, 28 insertions(+), 16 deletions(-) create mode 100644 myems-api/run.sh diff --git a/database/create_database_ubuntu.sh b/database/create_database_ubuntu.sh index ab4063bb..48e19429 100644 --- a/database/create_database_ubuntu.sh +++ b/database/create_database_ubuntu.sh @@ -11,4 +11,4 @@ mysql -uroot -p$mypassword < myems_historical_db.sql mysql -uroot -p$mypassword < myems_reporting_db.sql mysql -uroot -p$mypassword < myems_system_db.sql mysql -uroot -p$mypassword < myems_user_db.sql -mysql -uroot -p$mypassword < demo/myems_system_db.sql \ No newline at end of file +mysql -uroot -p$mypassword < demo-en/myems_system_db.sql \ No newline at end of file diff --git a/myems-api/README.md b/myems-api/README.md index bedde402..c01659e9 100644 --- a/myems-api/README.md +++ b/myems-api/README.md @@ -19,7 +19,13 @@ gunicorn openpyxl - +## Quick Installation on Ubuntu 20.04 +We're using anaconda python and create a virtual environment named "myems". +```bash +conda activate myems +pip install -r requirements.txt +run.sh +``` ## Installation In this step, you will install myems-api on Ubuntu for production or development. diff --git a/myems-api/config.py b/myems-api/config.py index b762986a..c8567c6f 100644 --- a/myems-api/config.py +++ b/myems-api/config.py @@ -1,62 +1,62 @@ myems_system_db = { 'user': 'root', - 'password': '!MyEMS1', + 'password': 'pAssw0rd@12345', 'host': '127.0.0.1', 'database': 'myems_system_db', } myems_energy_db = { 'user': 'root', - 'password': '!MyEMS1', + 'password': 'pAssw0rd@12345', 'host': '127.0.0.1', 'database': 'myems_energy_db', } myems_energy_baseline_db = { 'user': 'root', - 'password': '!MyEMS1', + 'password': 'pAssw0rd@12345', 'host': '127.0.0.1', 'database': 'myems_energy_baseline_db', } myems_billing_db = { 'user': 'root', - 'password': '!MyEMS1', + 'password': 'pAssw0rd@12345', 'host': '127.0.0.1', 'database': 'myems_billing_db', } myems_billing_baseline_db = { 'user': 'root', - 'password': '!MyEMS1', + 'password': 'pAssw0rd@12345', 'host': '127.0.0.1', 'database': 'myems_billing_baseline_db', } myems_historical_db = { 'user': 'root', - 'password': '!MyEMS1', + 'password': 'pAssw0rd@12345', 'host': '127.0.0.1', 'database': 'myems_historical_db', } myems_user_db = { 'user': 'root', - 'password': '!MyEMS1', + 'password': 'pAssw0rd@12345', 'host': '127.0.0.1', 'database': 'myems_user_db', } myems_fdd_db = { 'user': 'root', - 'password': '!MyEMS1', + 'password': 'pAssw0rd@12345', 'host': '127.0.0.1', 'database': 'myems_fdd_db', } myems_reporting_db = { 'user': 'root', - 'password': '!MyEMS1', + 'password': 'pAssw0rd@12345', 'host': '127.0.0.1', 'database': 'myems_reporting_db', } diff --git a/myems-api/myems-api.service b/myems-api/myems-api.service index 2bc26c4e..221ba9c7 100644 --- a/myems-api/myems-api.service +++ b/myems-api/myems-api.service @@ -5,10 +5,10 @@ After=network.target [Service] PIDFile=/run/myems-api/pid -User=root -Group=root -WorkingDirectory=/myems-api -ExecStart=/usr/local/bin/gunicorn --pid /run/myems-api/pid --timeout 600 --workers=4 app:api +User=nvidia +Group=nvidia +WorkingDirectory=/home/nvidia/PycharmProjects/myems/myems-api +ExecStart=/home/nvidia/anaconda3/envs/myems/bin/gunicorn --pid /run/myems-api/pid --timeout 600 --workers=4 app:api ExecReload=/bin/kill -s HUP $MAINPID ExecStop=/bin/kill -s TERM $MAINPID PrivateTmp=true diff --git a/myems-api/requirements.txt b/myems-api/requirements.txt index 12bd96f0..756cb6e6 100644 --- a/myems-api/requirements.txt +++ b/myems-api/requirements.txt @@ -1,9 +1,12 @@ anytree simplejson +mysql-connector-python==8.0.20 mysql.connector falcon falcon_cors falcon-multipart gunicorn -openpyxl +et_xmlfile==1.1 +jdcal +openpyxl==3.0.7 pillow \ No newline at end of file diff --git a/myems-api/run.sh b/myems-api/run.sh new file mode 100644 index 00000000..e4aa9d3f --- /dev/null +++ b/myems-api/run.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +gunicorn --pid pid --timeout 600 --workers=4 app:api \ No newline at end of file From 60906080f4b0daf038f323d69a1bd3b2313a454a Mon Sep 17 00:00:00 2001 From: winnerineast Date: Fri, 1 Oct 2021 18:58:05 +0800 Subject: [PATCH 3/7] mask out all clear password into encrypted ones. --- .gitignore | 3 ++- database/create_database_ubuntu.sh | 4 ++- myems-api/.gitignore | 1 + myems-api/config.py | 42 +++++++++++++++++------------- myems-api/requirements.txt | 3 ++- 5 files changed, 32 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 9a3a0b4a..f1124eec 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,5 @@ logs Thumbs.db # Folder config file -Desktop.ini \ No newline at end of file +Desktop.ini +/database/.env diff --git a/database/create_database_ubuntu.sh b/database/create_database_ubuntu.sh index 48e19429..31e43ee6 100644 --- a/database/create_database_ubuntu.sh +++ b/database/create_database_ubuntu.sh @@ -1,6 +1,8 @@ #!/bin/sh -mypassword="pAssw0rd@12345" +e_mypassword="U2FsdGVkX1+VZdIDGQqLkpIvF47w7nq+UyZAetppy2k=" + +mypassword=`echo $e_mypassword |openssl enc -aes-256-cbc -md sha512 -a -d -pbkdf2 -iter 100000 -salt -pass pass:$mypass` mysql -uroot -p$mypassword < myems_billing_baseline_db.sql mysql -uroot -p$mypassword < myems_billing_db.sql diff --git a/myems-api/.gitignore b/myems-api/.gitignore index 83d71500..797f69bb 100644 --- a/myems-api/.gitignore +++ b/myems-api/.gitignore @@ -50,3 +50,4 @@ doc/_build *~ advancereport/ +/.env diff --git a/myems-api/config.py b/myems-api/config.py index c8567c6f..a9a7f317 100644 --- a/myems-api/config.py +++ b/myems-api/config.py @@ -1,62 +1,68 @@ +from decouple import config + + +userID = config('userID', default='root') +password = config('password', default='') + myems_system_db = { - 'user': 'root', - 'password': 'pAssw0rd@12345', + 'user': userID, + 'password': password, 'host': '127.0.0.1', 'database': 'myems_system_db', } myems_energy_db = { - 'user': 'root', - 'password': 'pAssw0rd@12345', + 'user': userID, + 'password': password, 'host': '127.0.0.1', 'database': 'myems_energy_db', } myems_energy_baseline_db = { - 'user': 'root', - 'password': 'pAssw0rd@12345', + 'user': userID, + 'password': password, 'host': '127.0.0.1', 'database': 'myems_energy_baseline_db', } myems_billing_db = { - 'user': 'root', - 'password': 'pAssw0rd@12345', + 'user': userID, + 'password': password, 'host': '127.0.0.1', 'database': 'myems_billing_db', } myems_billing_baseline_db = { - 'user': 'root', - 'password': 'pAssw0rd@12345', + 'user': userID, + 'password': password, 'host': '127.0.0.1', 'database': 'myems_billing_baseline_db', } myems_historical_db = { - 'user': 'root', - 'password': 'pAssw0rd@12345', + 'user': userID, + 'password': password, 'host': '127.0.0.1', 'database': 'myems_historical_db', } myems_user_db = { - 'user': 'root', - 'password': 'pAssw0rd@12345', + 'user': userID, + 'password': password, 'host': '127.0.0.1', 'database': 'myems_user_db', } myems_fdd_db = { - 'user': 'root', - 'password': 'pAssw0rd@12345', + 'user': userID, + 'password': password, 'host': '127.0.0.1', 'database': 'myems_fdd_db', } myems_reporting_db = { - 'user': 'root', - 'password': 'pAssw0rd@12345', + 'user': userID, + 'password': password, 'host': '127.0.0.1', 'database': 'myems_reporting_db', } diff --git a/myems-api/requirements.txt b/myems-api/requirements.txt index 756cb6e6..cbcd0069 100644 --- a/myems-api/requirements.txt +++ b/myems-api/requirements.txt @@ -9,4 +9,5 @@ gunicorn et_xmlfile==1.1 jdcal openpyxl==3.0.7 -pillow \ No newline at end of file +pillow +python-decouple \ No newline at end of file From 84abff5455be8dc5d299afe7e8b171d1b28e48b1 Mon Sep 17 00:00:00 2001 From: winnerineast Date: Fri, 1 Oct 2021 19:02:50 +0800 Subject: [PATCH 4/7] mask out all clear password into encrypted ones for myems-cleaning. --- myems-cleaning/config.py | 14 ++++++++++---- myems-cleaning/requirements.txt | 4 +++- myems-cleaning/run.sh | 1 + 3 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 myems-cleaning/run.sh diff --git a/myems-cleaning/config.py b/myems-cleaning/config.py index b2eb0384..b41b45a6 100644 --- a/myems-cleaning/config.py +++ b/myems-cleaning/config.py @@ -1,14 +1,20 @@ +from decouple import config + + +userID = config('userID', default='root') +password = config('password', default='') + myems_system_db = { - 'user': 'root', - 'password': '!MyEMS1', + 'user': userID, + 'password': password, 'host': '127.0.0.1', 'database': 'myems_system_db', 'port': 3306, } myems_historical_db = { - 'user': 'root', - 'password': '!MyEMS1', + 'user': userID, + 'password': password, 'host': '127.0.0.1', 'database': 'myems_historical_db', 'port': 3306, diff --git a/myems-cleaning/requirements.txt b/myems-cleaning/requirements.txt index 51fbac7f..fce690d4 100644 --- a/myems-cleaning/requirements.txt +++ b/myems-cleaning/requirements.txt @@ -1,2 +1,4 @@ mysql-connector -schedule \ No newline at end of file +mysql-connector-python==8.0.20 +schedule +python-decouple \ No newline at end of file diff --git a/myems-cleaning/run.sh b/myems-cleaning/run.sh new file mode 100644 index 00000000..7a84f979 --- /dev/null +++ b/myems-cleaning/run.sh @@ -0,0 +1 @@ +python main.py \ No newline at end of file From 808addf1ecf45241b4bc599ab10a6dadf00a0aa6 Mon Sep 17 00:00:00 2001 From: winnerineast Date: Fri, 1 Oct 2021 19:05:48 +0800 Subject: [PATCH 5/7] mask out all clear password into encrypted ones for myems-modbus. --- myems-modbus-tcp/config.py | 14 ++++++++++---- myems-modbus-tcp/requirements.txt | 1 + myems-modbus-tcp/run.sh | 1 + 3 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 myems-modbus-tcp/run.sh diff --git a/myems-modbus-tcp/config.py b/myems-modbus-tcp/config.py index 212aaf51..b23123eb 100644 --- a/myems-modbus-tcp/config.py +++ b/myems-modbus-tcp/config.py @@ -1,14 +1,20 @@ +from decouple import config + + +userID = config('userID', default='root') +password = config('password', default='') + myems_system_db = { - 'user': 'root', - 'password': '!MyEMS1', + 'user': userID, + 'password': password, 'host': '127.0.0.1', 'database': 'myems_system_db', 'port': 3306, } myems_historical_db = { - 'user': 'root', - 'password': '!MyEMS1', + 'user': userID, + 'password': password, 'host': '127.0.0.1', 'database': 'myems_historical_db', 'port': 3306, diff --git a/myems-modbus-tcp/requirements.txt b/myems-modbus-tcp/requirements.txt index a867d987..f6bd7e95 100644 --- a/myems-modbus-tcp/requirements.txt +++ b/myems-modbus-tcp/requirements.txt @@ -1,3 +1,4 @@ mysql-connector modbus_tk schedule +python-decouple diff --git a/myems-modbus-tcp/run.sh b/myems-modbus-tcp/run.sh new file mode 100644 index 00000000..7a84f979 --- /dev/null +++ b/myems-modbus-tcp/run.sh @@ -0,0 +1 @@ +python main.py \ No newline at end of file From 3e884614cea2192dfd9ad5d63ad3808a7129e889 Mon Sep 17 00:00:00 2001 From: winnerineast Date: Fri, 1 Oct 2021 19:08:43 +0800 Subject: [PATCH 6/7] mask out all clear password into encrypted ones for myems-normalization. --- myems-modbus-tcp/requirements.txt | 1 + myems-normalization/config.py | 18 ++++++++++++------ myems-normalization/requirements.txt | 4 +++- myems-normalization/run.sh | 1 + 4 files changed, 17 insertions(+), 7 deletions(-) create mode 100644 myems-normalization/run.sh diff --git a/myems-modbus-tcp/requirements.txt b/myems-modbus-tcp/requirements.txt index f6bd7e95..784c235f 100644 --- a/myems-modbus-tcp/requirements.txt +++ b/myems-modbus-tcp/requirements.txt @@ -1,4 +1,5 @@ mysql-connector +mysql-connector-python==8.0.20 modbus_tk schedule python-decouple diff --git a/myems-normalization/config.py b/myems-normalization/config.py index ebfb65ce..49413d77 100644 --- a/myems-normalization/config.py +++ b/myems-normalization/config.py @@ -1,22 +1,28 @@ +from decouple import config + + +userID = config('userID', default='root') +password = config('password', default='') + myems_system_db = { - 'user': 'root', - 'password': '!MyEMS1', + 'user': userID, + 'password': password, 'host': '127.0.0.1', 'database': 'myems_system_db', 'port': 3306, } myems_energy_db = { - 'user': 'root', - 'password': '!MyEMS1', + 'user': userID, + 'password': password, 'host': '127.0.0.1', 'database': 'myems_energy_db', 'port': 3306, } myems_historical_db = { - 'user': 'root', - 'password': '!MyEMS1', + 'user': userID, + 'password': password, 'host': '127.0.0.1', 'database': 'myems_historical_db', 'port': 3306, diff --git a/myems-normalization/requirements.txt b/myems-normalization/requirements.txt index 67ece48e..919ce2d2 100644 --- a/myems-normalization/requirements.txt +++ b/myems-normalization/requirements.txt @@ -1,3 +1,5 @@ mysql-connector +mysql-connector-python==8.0.20 openpyxl -sympy \ No newline at end of file +sympy +python-decouple \ No newline at end of file diff --git a/myems-normalization/run.sh b/myems-normalization/run.sh new file mode 100644 index 00000000..7a84f979 --- /dev/null +++ b/myems-normalization/run.sh @@ -0,0 +1 @@ +python main.py \ No newline at end of file From b97c16fbb976431bd0df911066c3fb185b913416 Mon Sep 17 00:00:00 2001 From: "13621160019@163.com" <13621160019@163.com> Date: Tue, 12 Oct 2021 18:39:42 +0800 Subject: [PATCH 7/7] updated config.py files to move all variables to .env files via Python Decouple --- .gitignore | 1 - database/.gitignore | 45 ++++++++ database/README.md | 2 +- database/create_database_ubuntu.sh | 16 --- myems-aggregation/.env.example | 43 ++++++++ myems-aggregation/README.md | 31 ++++-- myems-aggregation/config.py | 76 ++++++++++---- myems-aggregation/requirements.txt | 3 +- myems-aggregation/run.sh | 1 + myems-api/.env.example | 82 +++++++++++++++ myems-api/.gitignore | 11 +- myems-api/README.md | 64 ++++++----- myems-api/config.py | 152 +++++++++++++++++++-------- myems-api/myems-api.service | 8 +- myems-api/requirements.txt | 7 +- myems-api/run.sh | 0 myems-cleaning/.env.example | 21 ++++ myems-cleaning/README.md | 32 ++++-- myems-cleaning/config.py | 41 +++++--- myems-cleaning/requirements.txt | 3 +- myems-modbus-tcp/.env.example | 21 ++++ myems-modbus-tcp/README.md | 23 ++-- myems-modbus-tcp/config.py | 44 +++++--- myems-modbus-tcp/requirements.txt | 3 +- myems-modbus-tcp/run.sh | 0 myems-normalization/.env.example | 46 ++++++++ myems-normalization/README.md | 47 +++++---- myems-normalization/config.py | 77 +++++++++----- myems-normalization/requirements.txt | 3 +- 29 files changed, 672 insertions(+), 231 deletions(-) create mode 100644 database/.gitignore delete mode 100644 database/create_database_ubuntu.sh create mode 100644 myems-aggregation/.env.example create mode 100755 myems-aggregation/run.sh create mode 100644 myems-api/.env.example mode change 100644 => 100755 myems-api/run.sh create mode 100644 myems-cleaning/.env.example create mode 100644 myems-modbus-tcp/.env.example mode change 100644 => 100755 myems-modbus-tcp/run.sh create mode 100644 myems-normalization/.env.example diff --git a/.gitignore b/.gitignore index f1124eec..31b9053f 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,3 @@ Thumbs.db # Folder config file Desktop.ini -/database/.env diff --git a/database/.gitignore b/database/.gitignore new file mode 100644 index 00000000..6f3370a5 --- /dev/null +++ b/database/.gitignore @@ -0,0 +1,45 @@ +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pythonenv* +# Logs +logs +*.log +*_build +*_static +*_templates +################# +## JetBrains +################# +.idea + +.vscode + +# System +.DS_Store + +# VIM swap files +.*.swp + +# VIM temp files +*~ + +################# +## Sublime Text +################# +*.sublime-* + +############ +## Windows +############ + +# Windows image file caches +Thumbs.db + +# Folder config file +Desktop.ini diff --git a/database/README.md b/database/README.md index cc472a23..2143a513 100644 --- a/database/README.md +++ b/database/README.md @@ -13,7 +13,7 @@ Providing database schema and scripts for MyEMS. ### Installation -Execute the following scripts in MySQL commandline, or execute in other MySQL client tools such as MySQL Workbench, Navicat, DBaver, phpMyAdmin, etc. Here is [an example shell script](create_database_ubuntu.sh) to initiate database in one command on Ubuntu 20.04. +Execute the following scripts in MySQL commandline, or execute in other MySQL client tools such as MySQL Workbench, Navicat, DBaver, phpMyAdmin, etc. ```bash cd myems/database/install mysql -u root -p < myems_billing_baseline_db.sql diff --git a/database/create_database_ubuntu.sh b/database/create_database_ubuntu.sh deleted file mode 100644 index 31e43ee6..00000000 --- a/database/create_database_ubuntu.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -e_mypassword="U2FsdGVkX1+VZdIDGQqLkpIvF47w7nq+UyZAetppy2k=" - -mypassword=`echo $e_mypassword |openssl enc -aes-256-cbc -md sha512 -a -d -pbkdf2 -iter 100000 -salt -pass pass:$mypass` - -mysql -uroot -p$mypassword < myems_billing_baseline_db.sql -mysql -uroot -p$mypassword < myems_billing_db.sql -mysql -uroot -p$mypassword < myems_energy_baseline_db.sql -mysql -uroot -p$mypassword < myems_energy_db.sql -mysql -uroot -p$mypassword < myems_fdd_db.sql -mysql -uroot -p$mypassword < myems_historical_db.sql -mysql -uroot -p$mypassword < myems_reporting_db.sql -mysql -uroot -p$mypassword < myems_system_db.sql -mysql -uroot -p$mypassword < myems_user_db.sql -mysql -uroot -p$mypassword < demo-en/myems_system_db.sql \ No newline at end of file diff --git a/myems-aggregation/.env.example b/myems-aggregation/.env.example new file mode 100644 index 00000000..80de7464 --- /dev/null +++ b/myems-aggregation/.env.example @@ -0,0 +1,43 @@ +# config for myems_system_db +MYEMS_SYSTEM_DB_HOST=127.0.0.1 +MYEMS_SYSTEM_DB_PORT=3306 +MYEMS_SYSTEM_DB_DATABASE=myems_system_db +MYEMS_SYSTEM_DB_USER=root +MYEMS_SYSTEM_DB_PASSWORD=!MyEMS1 + +# config for myems_historical_db +MYEMS_HISTORICAL_DB_HOST=127.0.0.1 +MYEMS_HISTORICAL_DB_PORT=3306 +MYEMS_HISTORICAL_DB_DATABASE=myems_historical_db +MYEMS_HISTORICAL_DB_USER=root +MYEMS_HISTORICAL_DB_PASSWORD=!MyEMS1 + +# config for myems_energy_db +MYEMS_ENERGY_DB_HOST=127.0.0.1 +MYEMS_ENERGY_DB_PORT=3306 +MYEMS_ENERGY_DB_DATABASE=myems_energy_db +MYEMS_ENERGY_DB_USER=root +MYEMS_ENERGY_DB_PASSWORD=!MyEMS1 + +# config for myems_billing_db +MYEMS_BILLING_DB_HOST=127.0.0.1 +MYEMS_BILLING_DB_PORT=3306 +MYEMS_BILLING_DB_DATABASE=myems_billing_db +MYEMS_BILLING_DB_USER=root +MYEMS_BILLING_DB_PASSWORD=!MyEMS1 + +# indicates how long in minutes energy data will be aggregated +# 30 for half hourly +# 60 for hourly +MINUTES_TO_COUNT=60 + +# indicates from when (in UTC timezone) to recalculate if the energy data is null or were cleared +# format string: "%Y-%m-%d %H:%M:%S" +START_DATETIME_UTC="2019-12-31 16:00:00" + +# indicates the project's time zone offset from UTC +UTC_OFFSET=+08:00 + +# the number of worker processes in parallel +# the pool size depends on the computing performance of the database server and the analysis server +POOL_SIZE=5 \ No newline at end of file diff --git a/myems-aggregation/README.md b/myems-aggregation/README.md index ae0c6262..4a54a46d 100644 --- a/myems-aggregation/README.md +++ b/myems-aggregation/README.md @@ -6,7 +6,17 @@ This service is a component of MyEMS and it aggregates normalized data up to mul ### Prerequisites -mysql.connector +mysql-connector-python + +python-decouple + +### Quick Run for Development + +```bash +pip install -r requirements.txt +chmod +x run.sh +run.sh +``` ### Installation @@ -16,7 +26,7 @@ cd ~/tools wget https://dev.mysql.com/get/Downloads/Connector-Python/mysql-connector-python-8.0.20.tar.gz tar xzf mysql-connector-python-8.0.20.tar.gz cd ~/tools/mysql-connector-python-8.0.20 -sudo python3 setup.py install +python3 setup.py install ``` Install myems-aggregation service: @@ -24,28 +34,29 @@ Install myems-aggregation service: cd ~ git clone https://github.com/MyEMS/myems.git cd myems -sudo git checkout master (or the latest release tag) -sudo cp -R ~/myems/myems-aggregation /myems-aggregation +git checkout master (or the latest release tag) +cp -R ~/myems/myems-aggregation /myems-aggregation ``` -Edit config.py +Create .env file based on .env.example and edit the .env file if needed: ```bash -sudo nano /myems-aggregation/config.py +cp /myems-aggregation/.env.example /myems-aggregation/.env +nano /myems-aggregation/.env ``` Setup systemd service: ```bash -sudo cp myems-aggregation.service /lib/systemd/system/ +cp myems-aggregation.service /lib/systemd/system/ ``` Enable the service: ```bash -sudo systemctl enable myems-aggregation.service +systemctl enable myems-aggregation.service ``` Start the service: ```bash -sudo systemctl start myems-aggregation.service +systemctl start myems-aggregation.service ``` Monitor the service: ```bash -sudo systemctl status myems-aggregation.service +systemctl status myems-aggregation.service ``` View the log: ```bash diff --git a/myems-aggregation/config.py b/myems-aggregation/config.py index cddd91e1..3d63f55c 100644 --- a/myems-aggregation/config.py +++ b/myems-aggregation/config.py @@ -1,45 +1,79 @@ +from decouple import config + +MYEMS_SYSTEM_DB_HOST = config('MYEMS_SYSTEM_DB_HOST', default='127.0.0.1') +MYEMS_SYSTEM_DB_PORT = config('MYEMS_SYSTEM_DB_PORT', default=3306, cast=int) +MYEMS_SYSTEM_DB_DATABASE = config('MYEMS_SYSTEM_DB_DATABASE', default='myems_system_db') +MYEMS_SYSTEM_DB_USER = config('MYEMS_SYSTEM_DB_USER', default='root') +MYEMS_SYSTEM_DB_PASSWORD = config('MYEMS_SYSTEM_DB_PASSWORD', default='!MyEMS1') + +MYEMS_HISTORICAL_DB_HOST = config('MYEMS_HISTORICAL_DB_HOST', default='127.0.0.1') +MYEMS_HISTORICAL_DB_PORT = config('MYEMS_HISTORICAL_DB_PORT', default=3306, cast=int) +MYEMS_HISTORICAL_DB_DATABASE = config('MYEMS_HISTORICAL_DB_DATABASE', default='myems_system_db') +MYEMS_HISTORICAL_DB_USER = config('MYEMS_HISTORICAL_DB_USER', default='root') +MYEMS_HISTORICAL_DB_PASSWORD = config('MYEMS_HISTORICAL_DB_PASSWORD', default='!MyEMS1') + +MYEMS_ENERGY_DB_HOST = config('MYEMS_ENERGY_DB_HOST', default='127.0.0.1') +MYEMS_ENERGY_DB_PORT = config('MYEMS_ENERGY_DB_PORT', default=3306, cast=int) +MYEMS_ENERGY_DB_DATABASE = config('MYEMS_ENERGY_DB_DATABASE', default='myems_system_db') +MYEMS_ENERGY_DB_USER = config('MYEMS_ENERGY_DB_USER', default='root') +MYEMS_ENERGY_DB_PASSWORD = config('MYEMS_ENERGY_DB_PASSWORD', default='!MyEMS1') + +MYEMS_BILLING_DB_HOST = config('MYEMS_BILLING_DB_HOST', default='127.0.0.1') +MYEMS_BILLING_DB_PORT = config('MYEMS_BILLING_DB_PORT', default=3306, cast=int) +MYEMS_BILLING_DB_DATABASE = config('MYEMS_BILLING_DB_DATABASE', default='myems_system_db') +MYEMS_BILLING_DB_USER = config('MYEMS_BILLING_DB_USER', default='root') +MYEMS_BILLING_DB_PASSWORD = config('MYEMS_BILLING_DB_PASSWORD', default='!MyEMS1') + +MINUTES_TO_COUNT = config('MINUTES_TO_COUNT', default=60, cast=int) +START_DATETIME_UTC = config('START_DATETIME_UTC', default='2019-12-31 16:00:00') +UTC_OFFSET = config('UTC_OFFSET', default='+08:00') +POOL_SIZE = config('POOL_SIZE', default=5, cast=int) + myems_system_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_system_db', + 'host': MYEMS_SYSTEM_DB_HOST, + 'port': MYEMS_SYSTEM_DB_PORT, + 'database': MYEMS_SYSTEM_DB_DATABASE, + 'user': MYEMS_SYSTEM_DB_USER, + 'password': MYEMS_SYSTEM_DB_PASSWORD, } myems_historical_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_historical_db', + 'host': MYEMS_HISTORICAL_DB_HOST, + 'port': MYEMS_HISTORICAL_DB_PORT, + 'database': MYEMS_HISTORICAL_DB_DATABASE, + 'user': MYEMS_HISTORICAL_DB_USER, + 'password': MYEMS_HISTORICAL_DB_PASSWORD, } myems_energy_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_energy_db', + 'host': MYEMS_ENERGY_DB_HOST, + 'port': MYEMS_ENERGY_DB_PORT, + 'database': MYEMS_ENERGY_DB_DATABASE, + 'user': MYEMS_ENERGY_DB_USER, + 'password': MYEMS_ENERGY_DB_PASSWORD, } myems_billing_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_billing_db', + 'host': MYEMS_BILLING_DB_HOST, + 'port': MYEMS_BILLING_DB_PORT, + 'database': MYEMS_BILLING_DB_DATABASE, + 'user': MYEMS_BILLING_DB_USER, + 'password': MYEMS_BILLING_DB_PASSWORD, } - # indicates how long in minutes energy data will be aggregated # 30 for half hourly # 60 for hourly -minutes_to_count = 60 +minutes_to_count = MINUTES_TO_COUNT # indicates from when (in UTC timezone) to recalculate if the energy data is null or were cleared # format string: '%Y-%m-%d %H:%M:%S' -start_datetime_utc = '2019-12-31 16:00:00' +start_datetime_utc = START_DATETIME_UTC # indicates the project's time zone offset from UTC -utc_offset = '+08:00' +utc_offset = UTC_OFFSET # the number of worker processes in parallel # the pool size depends on the computing performance of the database server and the analysis server -pool_size = 5 +pool_size = POOL_SIZE diff --git a/myems-aggregation/requirements.txt b/myems-aggregation/requirements.txt index a6dc3627..c4beab52 100644 --- a/myems-aggregation/requirements.txt +++ b/myems-aggregation/requirements.txt @@ -1 +1,2 @@ -mysql-connector \ No newline at end of file +mysql-connector-python +python-decouple \ No newline at end of file diff --git a/myems-aggregation/run.sh b/myems-aggregation/run.sh new file mode 100755 index 00000000..cbc5f14c --- /dev/null +++ b/myems-aggregation/run.sh @@ -0,0 +1 @@ +python3 main.py \ No newline at end of file diff --git a/myems-api/.env.example b/myems-api/.env.example new file mode 100644 index 00000000..f4c2db4a --- /dev/null +++ b/myems-api/.env.example @@ -0,0 +1,82 @@ +# config for myems_system_db +MYEMS_SYSTEM_DB_HOST=127.0.0.1 +MYEMS_SYSTEM_DB_PORT=3306 +MYEMS_SYSTEM_DB_DATABASE=myems_system_db +MYEMS_SYSTEM_DB_USER=root +MYEMS_SYSTEM_DB_PASSWORD=!MyEMS1 + +# config for myems_energy_db +MYEMS_ENERGY_DB_HOST=127.0.0.1 +MYEMS_ENERGY_DB_PORT=3306 +MYEMS_ENERGY_DB_DATABASE=myems_energy_db +MYEMS_ENERGY_DB_USER=root +MYEMS_ENERGY_DB_PASSWORD=!MyEMS1 + +# config for myems_energy_baseline_db +MYEMS_ENERGY_BASELINE_DB_HOST=127.0.0.1 +MYEMS_ENERGY_BASELINE_DB_PORT=3306 +MYEMS_ENERGY_BASELINE_DB_DATABASE=myems_energy_baseline_db +MYEMS_ENERGY_BASELINE_DB_USER=root +MYEMS_ENERGY_BASELINE_DB_PASSWORD=!MyEMS1 + +# config for myems_billing_db +MYEMS_BILLING_DB_HOST=127.0.0.1 +MYEMS_BILLING_DB_PORT=3306 +MYEMS_BILLING_DB_DATABASE=myems_billing_db +MYEMS_BILLING_DB_USER=root +MYEMS_BILLING_DB_PASSWORD=!MyEMS1 + +# config for myems_billing_baseline_db +MYEMS_BILLING_BASELINE_DB_HOST=127.0.0.1 +MYEMS_BILLING_BASELINE_DB_PORT=3306 +MYEMS_BILLING_BASELINE_DB_DATABASE=myems_billing_baseline_db +MYEMS_BILLING_BASELINE_DB_USER=root +MYEMS_BILLING_BASELINE_DB_PASSWORD=!MyEMS1 + +# config for myems_historical_db +MYEMS_HISTORICAL_DB_HOST=127.0.0.1 +MYEMS_HISTORICAL_DB_PORT=3306 +MYEMS_HISTORICAL_DB_DATABASE=myems_historical_db +MYEMS_HISTORICAL_DB_USER=root +MYEMS_HISTORICAL_DB_PASSWORD=!MyEMS1 + +# config for myems_user_db +MYEMS_USER_DB_HOST=127.0.0.1 +MYEMS_USER_DB_PORT=3306 +MYEMS_USER_DB_DATABASE=myems_user_db +MYEMS_USER_DB_USER=root +MYEMS_USER_DB_PASSWORD=!MyEMS1 + +# config for myems_fdd_db +MYEMS_FDD_DB_HOST=127.0.0.1 +MYEMS_FDD_DB_PORT=3306 +MYEMS_FDD_DB_DATABASE=myems_fdd_db +MYEMS_FDD_DB_USER=root +MYEMS_FDD_DB_PASSWORD=!MyEMS1 + +# config for myems_reporting_db +MYEMS_REPORTING_DB_HOST=127.0.0.1 +MYEMS_REPORTING_DB_PORT=3306 +MYEMS_REPORTING_DB_DATABASE=myems_reporting_db +MYEMS_REPORTING_DB_USER=root +MYEMS_REPORTING_DB_PASSWORD=!MyEMS1 + +# indicated in how many minutes to calculate meter energy consumption +# 30 for half hourly period +# 60 for hourly period +MINUTES_TO_COUNT=60 + +# indicates the project's time zone offset from UTC +UTC_OFFSET=+08:00 + +# indicates from when ( in local timezone) of the day to calculate working days +WORKING_DAY_START_TIME_LOCAL=00:00:00 + +# indicates where user uploaded files will be saved to +# must use the root folder of myems-admin web application +# for example if you serve myems-admin at /var/www/html/admin +# you should set the upload_path as below +UPLOAD_PATH=/var/www/html/admin/upload/ + +# main currency unit +CURRENCY_UNIT=CNY \ No newline at end of file diff --git a/myems-api/.gitignore b/myems-api/.gitignore index 797f69bb..4b3fcb2b 100644 --- a/myems-api/.gitignore +++ b/myems-api/.gitignore @@ -50,4 +50,13 @@ doc/_build *~ advancereport/ -/.env + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pythonenv* diff --git a/myems-api/README.md b/myems-api/README.md index c01659e9..3e3be42b 100644 --- a/myems-api/README.md +++ b/myems-api/README.md @@ -19,13 +19,14 @@ gunicorn openpyxl -## Quick Installation on Ubuntu 20.04 -We're using anaconda python and create a virtual environment named "myems". +## Quick Run for Development + ```bash -conda activate myems pip install -r requirements.txt +chmod +x run.sh run.sh ``` + ## Installation In this step, you will install myems-api on Ubuntu for production or development. @@ -37,7 +38,7 @@ For macOS developers, please refer to [Installation on macOS (Chinese)](./instal cd ~/tools git clone https://github.com/c0fec0de/anytree.git cd anytree -sudo python3 setup.py install +python3 setup.py install ``` * Install simplejson @@ -45,7 +46,7 @@ sudo python3 setup.py install cd ~/tools git clone https://github.com/simplejson/simplejson.git cd simplejson -sudo python3 setup.py install +python3 setup.py install ``` * Install MySQL Connector @@ -54,7 +55,7 @@ sudo python3 setup.py install wget https://dev.mysql.com/get/Downloads/Connector-Python/mysql-connector-python-8.0.20.tar.gz tar xzf mysql-connector-python-8.0.20.tar.gz cd ~/tools/mysql-connector-python-8.0.20 - sudo python3 setup.py install + python3 setup.py install ``` * Install Falcon, @@ -73,15 +74,15 @@ sudo python3 setup.py install pip3 download cython falcon falcon-cors falcon-multipart export LC_ALL="en_US.UTF-8" export LC_CTYPE="en_US.UTF-8" - sudo dpkg-reconfigure locales - sudo pip3 install --upgrade --no-index --find-links ~/tools/falcon cython falcon falcon-cors falcon-multipart + dpkg-reconfigure locales + pip3 install --upgrade --no-index --find-links ~/tools/falcon cython falcon falcon-cors falcon-multipart ``` * Install gunicorn, refer to http://gunicorn.org ```bash mkdir ~/tools/gunicorn && cd ~/tools/gunicorn pip3 download gunicorn - sudo pip3 install --no-index --find-links ~/tools/gunicorn gunicorn + pip3 install --no-index --find-links ~/tools/gunicorn gunicorn ``` * Install openpyxl, refer to https://foss.heptapod.net/openpyxl/openpyxl @@ -110,12 +111,12 @@ tar xzf openpyxl-3.0.7.tar.gz ```bash cd ~/tools/et_xmlfile-1.1 -sudo python3 setup.py install +python3 setup.py install cd ~/tools/jdcal -sudo python3 setup.py install -sudo pip3 install --no-index --find-links ~/tools/pillow Pillow +python3 setup.py install +pip3 install --no-index --find-links ~/tools/pillow Pillow cd ~/tools/openpyxl-3.0.7 -sudo python3 setup.py install +python3 setup.py install ``` * Download myems: @@ -127,43 +128,38 @@ git clone https://github.com/MyEMS/myems.git * Install gunicorn service for myems-api: ```bash cd ~/myems/myems-api -sudo cp -R ~/myems/myems-api /myems-api +cp -R ~/myems/myems-api /myems-api ``` - Change IP address in the config file: +Create .env file based on .env.example and edit the .env file if needed: ```bash -sudo nano /myems-api/config.py +cp /myems-api/.env.example /myems-api/.env +nano /myems-api/.env ``` - Change the listening port (default is 8000) in gunicorn.socket: +Change the listening port (default is 8000) in gunicorn.socket: ```bash -sudo nano /myems-api/myems-api.socket +nano /myems-api/myems-api.socket ``` ```bash ListenStream=0.0.0.0:8000 ``` ```bash -sudo ufw allow 8000 +ufw allow 8000 ``` - Setup systemd configure files: +Setup systemd configure files: ```bash - sudo cp /myems-api/myems-api.service /lib/systemd/system/ - sudo cp /myems-api/myems-api.socket /lib/systemd/system/ - sudo cp /myems-api/myems-api.conf /usr/lib/tmpfiles.d/ +cp /myems-api/myems-api.service /lib/systemd/system/ +cp /myems-api/myems-api.socket /lib/systemd/system/ +cp /myems-api/myems-api.conf /usr/lib/tmpfiles.d/ ``` Next enable the services so that they autostart at boot: ```bash - sudo systemctl enable myems-api.socket - sudo systemctl enable myems-api.service + systemctl enable myems-api.socket + systemctl enable myems-api.service ``` - Start the services : +Start the services : ```bash - sudo systemctl start myems-api.socket - sudo systemctl start myems-api.service -``` - -## Run for debugging and testing -```bash -cd myems-api -sudo gunicorn -b 127.0.0.1:8000 app:api +systemctl start myems-api.socket +systemctl start myems-api.service ``` ## API List diff --git a/myems-api/config.py b/myems-api/config.py index a9a7f317..29b019ae 100644 --- a/myems-api/config.py +++ b/myems-api/config.py @@ -1,88 +1,154 @@ from decouple import config -userID = config('userID', default='root') -password = config('password', default='') +MYEMS_SYSTEM_DB_HOST = config('MYEMS_SYSTEM_DB_HOST', default='127.0.0.1') +MYEMS_SYSTEM_DB_PORT = config('MYEMS_SYSTEM_DB_PORT', default=3306, cast=int) +MYEMS_SYSTEM_DB_DATABASE = config('MYEMS_SYSTEM_DB_DATABASE', default='myems_system_db') +MYEMS_SYSTEM_DB_USER = config('MYEMS_SYSTEM_DB_USER', default='root') +MYEMS_SYSTEM_DB_PASSWORD = config('MYEMS_SYSTEM_DB_PASSWORD', default='!MyEMS1') + +MYEMS_ENERGY_DB_HOST = config('MYEMS_ENERGY_DB_HOST', default='127.0.0.1') +MYEMS_ENERGY_DB_PORT = config('MYEMS_ENERGY_DB_PORT', default=3306, cast=int) +MYEMS_ENERGY_DB_DATABASE = config('MYEMS_ENERGY_DB_DATABASE', default='myems_energy_db') +MYEMS_ENERGY_DB_USER = config('MYEMS_ENERGY_DB_USER', default='root') +MYEMS_ENERGY_DB_PASSWORD = config('MYEMS_ENERGY_DB_PASSWORD', default='!MyEMS1') + +MYEMS_ENERGY_BASELINE_DB_HOST = config('MYEMS_ENERGY_BASELINE_DB_HOST', default='127.0.0.1') +MYEMS_ENERGY_BASELINE_DB_PORT = config('MYEMS_ENERGY_BASELINE_DB_PORT', default=3306, cast=int) +MYEMS_ENERGY_BASELINE_DB_DATABASE = config('MYEMS_ENERGY_BASELINE_DB_DATABASE', default='myems_energy_baseline_db') +MYEMS_ENERGY_BASELINE_DB_USER = config('MYEMS_ENERGY_BASELINE_DB_USER', default='root') +MYEMS_ENERGY_BASELINE_DB_PASSWORD = config('MYEMS_ENERGY_BASELINE_DB_PASSWORD', default='!MyEMS1') + +MYEMS_BILLING_DB_HOST = config('MYEMS_BILLING_DB_HOST', default='127.0.0.1') +MYEMS_BILLING_DB_PORT = config('MYEMS_BILLING_DB_PORT', default=3306, cast=int) +MYEMS_BILLING_DB_DATABASE = config('MYEMS_BILLING_DB_DATABASE', default='myems_billing_db') +MYEMS_BILLING_DB_USER = config('MYEMS_BILLING_DB_USER', default='root') +MYEMS_BILLING_DB_PASSWORD = config('MYEMS_BILLING_DB_PASSWORD', default='!MyEMS1') + +MYEMS_BILLING_BASELINE_DB_HOST = config('MYEMS_BILLING_BASELINE_DB_HOST', default='127.0.0.1') +MYEMS_BILLING_BASELINE_DB_PORT = config('MYEMS_BILLING_BASELINE_DB_PORT', default=3306, cast=int) +MYEMS_BILLING_BASELINE_DB_DATABASE = config('MYEMS_BILLING_BASELINE_DB_DATABASE', default='myems_billing_baseline_db') +MYEMS_BILLING_BASELINE_DB_USER = config('MYEMS_BILLING_BASELINE_DB_USER', default='root') +MYEMS_BILLING_BASELINE_DB_PASSWORD = config('MYEMS_BILLING_BASELINE_DB_PASSWORD', default='!MyEMS1') + +MYEMS_HISTORICAL_DB_HOST = config('MYEMS_HISTORICAL_DB_HOST', default='127.0.0.1') +MYEMS_HISTORICAL_DB_PORT = config('MYEMS_HISTORICAL_DB_PORT', default=3306, cast=int) +MYEMS_HISTORICAL_DB_DATABASE = config('MYEMS_HISTORICAL_DB_DATABASE', default='myems_historical_db') +MYEMS_HISTORICAL_DB_USER = config('MYEMS_HISTORICAL_DB_USER', default='root') +MYEMS_HISTORICAL_DB_PASSWORD = config('MYEMS_HISTORICAL_DB_PASSWORD', default='!MyEMS1') + +MYEMS_USER_DB_HOST = config('MYEMS_USER_DB_HOST', default='127.0.0.1') +MYEMS_USER_DB_PORT = config('MYEMS_USER_DB_PORT', default=3306, cast=int) +MYEMS_USER_DB_DATABASE = config('MYEMS_USER_DB_DATABASE', default='myems_user_db') +MYEMS_USER_DB_USER = config('MYEMS_USER_DB_USER', default='root') +MYEMS_USER_DB_PASSWORD = config('MYEMS_USER_DB_PASSWORD', default='!MyEMS1') + +MYEMS_FDD_DB_HOST = config('MYEMS_FDD_DB_HOST', default='127.0.0.1') +MYEMS_FDD_DB_PORT = config('MYEMS_FDD_DB_PORT', default=3306, cast=int) +MYEMS_FDD_DB_DATABASE = config('MYEMS_FDD_DB_DATABASE', default='myems_fdd_db') +MYEMS_FDD_DB_USER = config('MYEMS_FDD_DB_USER', default='root') +MYEMS_FDD_DB_PASSWORD = config('MYEMS_FDD_DB_PASSWORD', default='!MyEMS1') + +MYEMS_REPORTING_DB_HOST = config('MYEMS_REPORTING_DB_HOST', default='127.0.0.1') +MYEMS_REPORTING_DB_PORT = config('MYEMS_REPORTING_DB_PORT', default=3306, cast=int) +MYEMS_REPORTING_DB_DATABASE = config('MYEMS_REPORTING_DB_DATABASE', default='myems_reporting_db') +MYEMS_REPORTING_DB_USER = config('MYEMS_REPORTING_DB_USER', default='root') +MYEMS_REPORTING_DB_PASSWORD = config('MYEMS_REPORTING_DB_PASSWORD', default='!MyEMS1') + +MINUTES_TO_COUNT = config('MINUTES_TO_COUNT', default=60, cast=int) +UTC_OFFSET = config('UTC_OFFSET', default='+08:00') +WORKING_DAY_START_TIME_LOCAL = config('WORKING_DAY_START_TIME_LOCAL', default='00:00:00') +UPLOAD_PATH = config('UPLOAD_PATH', default='/var/www/html/admin/upload/') +CURRENCY_UNIT = config('CURRENCY_UNIT', default='CNY') myems_system_db = { - 'user': userID, - 'password': password, - 'host': '127.0.0.1', - 'database': 'myems_system_db', + 'host': MYEMS_SYSTEM_DB_HOST, + 'port': MYEMS_SYSTEM_DB_PORT, + 'database': MYEMS_SYSTEM_DB_DATABASE, + 'user': MYEMS_SYSTEM_DB_USER, + 'password': MYEMS_SYSTEM_DB_PASSWORD, } myems_energy_db = { - 'user': userID, - 'password': password, - 'host': '127.0.0.1', - 'database': 'myems_energy_db', + 'host': MYEMS_ENERGY_DB_HOST, + 'port': MYEMS_ENERGY_DB_PORT, + 'database': MYEMS_ENERGY_DB_DATABASE, + 'user': MYEMS_ENERGY_DB_USER, + 'password': MYEMS_ENERGY_DB_PASSWORD, } myems_energy_baseline_db = { - 'user': userID, - 'password': password, - 'host': '127.0.0.1', - 'database': 'myems_energy_baseline_db', + 'host': MYEMS_ENERGY_BASELINE_DB_HOST, + 'port': MYEMS_ENERGY_BASELINE_DB_PORT, + 'database': MYEMS_ENERGY_BASELINE_DB_DATABASE, + 'user': MYEMS_ENERGY_BASELINE_DB_USER, + 'password': MYEMS_ENERGY_BASELINE_DB_PASSWORD, } myems_billing_db = { - 'user': userID, - 'password': password, - 'host': '127.0.0.1', - 'database': 'myems_billing_db', + 'host': MYEMS_BILLING_DB_HOST, + 'port': MYEMS_BILLING_DB_PORT, + 'database': MYEMS_BILLING_DB_DATABASE, + 'user': MYEMS_BILLING_DB_USER, + 'password': MYEMS_BILLING_DB_PASSWORD, } myems_billing_baseline_db = { - 'user': userID, - 'password': password, - 'host': '127.0.0.1', - 'database': 'myems_billing_baseline_db', + 'host': MYEMS_BILLING_BASELINE_DB_HOST, + 'port': MYEMS_BILLING_BASELINE_DB_PORT, + 'database': MYEMS_BILLING_BASELINE_DB_DATABASE, + 'user': MYEMS_BILLING_BASELINE_DB_USER, + 'password': MYEMS_BILLING_BASELINE_DB_PASSWORD, } myems_historical_db = { - 'user': userID, - 'password': password, - 'host': '127.0.0.1', - 'database': 'myems_historical_db', + 'host': MYEMS_HISTORICAL_DB_HOST, + 'port': MYEMS_HISTORICAL_DB_PORT, + 'database': MYEMS_HISTORICAL_DB_DATABASE, + 'user': MYEMS_HISTORICAL_DB_USER, + 'password': MYEMS_HISTORICAL_DB_PASSWORD, } myems_user_db = { - 'user': userID, - 'password': password, - 'host': '127.0.0.1', - 'database': 'myems_user_db', + 'host': MYEMS_USER_DB_HOST, + 'port': MYEMS_USER_DB_PORT, + 'database': MYEMS_USER_DB_DATABASE, + 'user': MYEMS_USER_DB_USER, + 'password': MYEMS_USER_DB_PASSWORD, } myems_fdd_db = { - 'user': userID, - 'password': password, - 'host': '127.0.0.1', - 'database': 'myems_fdd_db', + 'host': MYEMS_FDD_DB_HOST, + 'port': MYEMS_FDD_DB_PORT, + 'database': MYEMS_FDD_DB_DATABASE, + 'user': MYEMS_FDD_DB_USER, + 'password': MYEMS_FDD_DB_PASSWORD, } myems_reporting_db = { - 'user': userID, - 'password': password, - 'host': '127.0.0.1', - 'database': 'myems_reporting_db', + 'host': MYEMS_REPORTING_DB_HOST, + 'port': MYEMS_REPORTING_DB_PORT, + 'database': MYEMS_REPORTING_DB_DATABASE, + 'user': MYEMS_REPORTING_DB_USER, + 'password': MYEMS_REPORTING_DB_PASSWORD, } # indicated in how many minutes to calculate meter energy consumption # 30 for half hourly period # 60 for hourly period -minutes_to_count = 60 +minutes_to_count = MINUTES_TO_COUNT # indicates the project's time zone offset from UTC -utc_offset = '+08:00' +utc_offset = UTC_OFFSET # indicates from when ( in local timezone) of the day to calculate working days -working_day_start_time_local = '00:00:00' +working_day_start_time_local = WORKING_DAY_START_TIME_LOCAL # indicates where user uploaded files will be saved to # must use the root folder of myems-admin web application # for example if you serve myems-admin at /var/www/html/admin # you should set the upload_path as below -upload_path = '/var/www/html/admin/upload/' +upload_path = UPLOAD_PATH # main currency unit -currency_unit = 'CNY' +currency_unit = CURRENCY_UNIT diff --git a/myems-api/myems-api.service b/myems-api/myems-api.service index 221ba9c7..2bc26c4e 100644 --- a/myems-api/myems-api.service +++ b/myems-api/myems-api.service @@ -5,10 +5,10 @@ After=network.target [Service] PIDFile=/run/myems-api/pid -User=nvidia -Group=nvidia -WorkingDirectory=/home/nvidia/PycharmProjects/myems/myems-api -ExecStart=/home/nvidia/anaconda3/envs/myems/bin/gunicorn --pid /run/myems-api/pid --timeout 600 --workers=4 app:api +User=root +Group=root +WorkingDirectory=/myems-api +ExecStart=/usr/local/bin/gunicorn --pid /run/myems-api/pid --timeout 600 --workers=4 app:api ExecReload=/bin/kill -s HUP $MAINPID ExecStop=/bin/kill -s TERM $MAINPID PrivateTmp=true diff --git a/myems-api/requirements.txt b/myems-api/requirements.txt index cbcd0069..b2db7617 100644 --- a/myems-api/requirements.txt +++ b/myems-api/requirements.txt @@ -1,13 +1,12 @@ anytree simplejson -mysql-connector-python==8.0.20 -mysql.connector +mysql-connector-python falcon falcon_cors falcon-multipart gunicorn -et_xmlfile==1.1 +et_xmlfile jdcal -openpyxl==3.0.7 +openpyxl pillow python-decouple \ No newline at end of file diff --git a/myems-api/run.sh b/myems-api/run.sh old mode 100644 new mode 100755 diff --git a/myems-cleaning/.env.example b/myems-cleaning/.env.example new file mode 100644 index 00000000..cd639b32 --- /dev/null +++ b/myems-cleaning/.env.example @@ -0,0 +1,21 @@ +# config for myems_system_db +MYEMS_SYSTEM_DB_HOST=127.0.0.1 +MYEMS_SYSTEM_DB_PORT=3306 +MYEMS_SYSTEM_DB_DATABASE=myems_system_db +MYEMS_SYSTEM_DB_USER=root +MYEMS_SYSTEM_DB_PASSWORD=!MyEMS1 + +# config for myems_historical_db +MYEMS_HISTORICAL_DB_HOST=127.0.0.1 +MYEMS_HISTORICAL_DB_PORT=3306 +MYEMS_HISTORICAL_DB_DATABASE=myems_historical_db +MYEMS_HISTORICAL_DB_USER=root +MYEMS_HISTORICAL_DB_PASSWORD=!MyEMS1 + +# indicates how long analog values and digital values will be kept in database +# the longer days the more memory and disc space needed. +# NOTE: By default, energy values in historical db will never be deleted automatically. +LIVE_IN_DAYS=365 + +# indicates if the program is in debug mode +IS_DEBUG=False \ No newline at end of file diff --git a/myems-cleaning/README.md b/myems-cleaning/README.md index 0ac571ab..605628eb 100644 --- a/myems-cleaning/README.md +++ b/myems-cleaning/README.md @@ -9,7 +9,18 @@ This service is a component of MyEMS and it cleans the historical data. ### Prerequisites -mysql.connector +mysql-connector-python + +schedule + +python-decouple + +### Quick Run for Development +```bash +pip install -r requirements.txt +chmod +x run.sh +run.sh +``` ### Installation @@ -19,7 +30,7 @@ cd ~/tools wget https://dev.mysql.com/get/Downloads/Connector-Python/mysql-connector-python-8.0.20.tar.gz tar xzf mysql-connector-python-8.0.20.tar.gz cd ~/tools/mysql-connector-python-8.0.20 -sudo python3 setup.py install +python3 setup.py install ``` Install myems-cleaning service @@ -27,28 +38,29 @@ Install myems-cleaning service cd ~ git clone https://github.com/MyEMS/myems.git cd myems -sudo git checkout master (or the latest release tag) -sudo cp -R ~/myems/myems-cleaning /myems-cleaning +git checkout master (or the latest release tag) +cp -R ~/myems/myems-cleaning /myems-cleaning ``` -Open config file and edit database configuration +Create .env file based on .env.example and edit the .env file if needed: ```bash -sudo nano /myems-cleaning/config.py +cp /myems-cleaning/.env.example /myems-cleaning/.env +nano /myems-cleaning/.env ``` Setup systemd service: ```bash -sudo cp myems-cleaning.service /lib/systemd/system/ +cp myems-cleaning.service /lib/systemd/system/ ``` Enable the service: ```bash -sudo systemctl enable myems-cleaning.service +systemctl enable myems-cleaning.service ``` Start the service: ```bash -sudo systemctl start myems-cleaning.service +systemctl start myems-cleaning.service ``` Monitor the service: ```bash -sudo systemctl status myems-cleaning.service +systemctl status myems-cleaning.service ``` View the log: ```bash diff --git a/myems-cleaning/config.py b/myems-cleaning/config.py index b41b45a6..dbaabef0 100644 --- a/myems-cleaning/config.py +++ b/myems-cleaning/config.py @@ -1,30 +1,41 @@ from decouple import config -userID = config('userID', default='root') -password = config('password', default='') +MYEMS_SYSTEM_DB_HOST = config('MYEMS_SYSTEM_DB_HOST', default='127.0.0.1') +MYEMS_SYSTEM_DB_PORT = config('MYEMS_SYSTEM_DB_PORT', default=3306, cast=int) +MYEMS_SYSTEM_DB_DATABASE = config('MYEMS_SYSTEM_DB_DATABASE', default='myems_system_db') +MYEMS_SYSTEM_DB_USER = config('MYEMS_SYSTEM_DB_USER', default='root') +MYEMS_SYSTEM_DB_PASSWORD = config('MYEMS_SYSTEM_DB_PASSWORD', default='!MyEMS1') + +MYEMS_HISTORICAL_DB_HOST = config('MYEMS_HISTORICAL_DB_HOST', default='127.0.0.1') +MYEMS_HISTORICAL_DB_PORT = config('MYEMS_HISTORICAL_DB_PORT', default=3306, cast=int) +MYEMS_HISTORICAL_DB_DATABASE = config('MYEMS_HISTORICAL_DB_DATABASE', default='myems_historical_db') +MYEMS_HISTORICAL_DB_USER = config('MYEMS_HISTORICAL_DB_USER', default='root') +MYEMS_HISTORICAL_DB_PASSWORD = config('MYEMS_HISTORICAL_DB_PASSWORD', default='!MyEMS1') + +LIVE_IN_DAYS = config('LIVE_IN_DAYS', default=365, cast=int) +IS_DEBUG = config('IS_DEBUG', default=False, cast=bool) myems_system_db = { - 'user': userID, - 'password': password, - 'host': '127.0.0.1', - 'database': 'myems_system_db', - 'port': 3306, + 'host': MYEMS_SYSTEM_DB_HOST, + 'port': MYEMS_SYSTEM_DB_PORT, + 'database': MYEMS_SYSTEM_DB_DATABASE, + 'user': MYEMS_SYSTEM_DB_USER, + 'password': MYEMS_SYSTEM_DB_PASSWORD, } myems_historical_db = { - 'user': userID, - 'password': password, - 'host': '127.0.0.1', - 'database': 'myems_historical_db', - 'port': 3306, + 'host': MYEMS_HISTORICAL_DB_HOST, + 'port': MYEMS_HISTORICAL_DB_PORT, + 'database': MYEMS_HISTORICAL_DB_DATABASE, + 'user': MYEMS_HISTORICAL_DB_USER, + 'password': MYEMS_HISTORICAL_DB_PASSWORD, } # indicates how long analog values and digital values will be kept in database # the longer days the more memory and disc space needed. -live_in_days = 365 - # NOTE: By default, energy values in historical db will never be deleted automatically. +live_in_days = LIVE_IN_DAYS # indicates if the program is in debug mode -is_debug = False +is_debug = IS_DEBUG diff --git a/myems-cleaning/requirements.txt b/myems-cleaning/requirements.txt index fce690d4..7ba59426 100644 --- a/myems-cleaning/requirements.txt +++ b/myems-cleaning/requirements.txt @@ -1,4 +1,3 @@ -mysql-connector -mysql-connector-python==8.0.20 +mysql-connector-python schedule python-decouple \ No newline at end of file diff --git a/myems-modbus-tcp/.env.example b/myems-modbus-tcp/.env.example new file mode 100644 index 00000000..8fc9a034 --- /dev/null +++ b/myems-modbus-tcp/.env.example @@ -0,0 +1,21 @@ +# config for myems_system_db +MYEMS_SYSTEM_DB_HOST=127.0.0.1 +MYEMS_SYSTEM_DB_PORT=3306 +MYEMS_SYSTEM_DB_DATABASE=myems_system_db +MYEMS_SYSTEM_DB_USER=root +MYEMS_SYSTEM_DB_PASSWORD=!MyEMS1 + +# config for myems_historical_db +MYEMS_HISTORICAL_DB_HOST=127.0.0.1 +MYEMS_HISTORICAL_DB_PORT=3306 +MYEMS_HISTORICAL_DB_DATABASE=myems_historical_db +MYEMS_HISTORICAL_DB_USER=root +MYEMS_HISTORICAL_DB_PASSWORD=!MyEMS1 + +# Indicates how long the process waits between readings +INTERVAL_IN_SECONDS=600 + +# Get the gateway ID and token from MyEMS Admin +# This is used for getting data sources associated with the gateway +GATEWAY_ID=1 +GATEWAY_TOKEN=AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA \ No newline at end of file diff --git a/myems-modbus-tcp/README.md b/myems-modbus-tcp/README.md index 088e5c65..1e0acc63 100644 --- a/myems-modbus-tcp/README.md +++ b/myems-modbus-tcp/README.md @@ -4,13 +4,23 @@ This service is a component of MyEMS to acquire data from Modbus TCP devices. ### Prerequisites -pyserial -modbus-tk +mysql-connector-python -mysql.connector +modbus_tk -Schedule +schedule + +python-decouple + + +### Quick Run for Development + +```bash +pip install -r requirements.txt +chmod +x run.sh +run.sh +``` ### Installation @@ -50,9 +60,10 @@ cd myems git checkout master (or the latest release tag) cp -R ~/myems/myems-modbus-tcp /myems-modbus-tcp ``` -Edit the config +Create .env file based on .env.example and edit the .env file if needed: ```bash -nano /myems-modbus-tcp/config.py +cp /myems-modbus-tcp/.env.example /myems-cleaning/.env +nano /myems-modbus-tcp/.env ``` Setup systemd service: ```bash diff --git a/myems-modbus-tcp/config.py b/myems-modbus-tcp/config.py index b23123eb..bfe85e2f 100644 --- a/myems-modbus-tcp/config.py +++ b/myems-modbus-tcp/config.py @@ -1,31 +1,45 @@ from decouple import config -userID = config('userID', default='root') -password = config('password', default='') +MYEMS_SYSTEM_DB_HOST = config('MYEMS_SYSTEM_DB_HOST', default='127.0.0.1') +MYEMS_SYSTEM_DB_PORT = config('MYEMS_SYSTEM_DB_PORT', default=3306, cast=int) +MYEMS_SYSTEM_DB_DATABASE = config('MYEMS_SYSTEM_DB_DATABASE', default='myems_system_db') +MYEMS_SYSTEM_DB_USER = config('MYEMS_SYSTEM_DB_USER', default='root') +MYEMS_SYSTEM_DB_PASSWORD = config('MYEMS_SYSTEM_DB_PASSWORD', default='!MyEMS1') + +MYEMS_HISTORICAL_DB_HOST = config('MYEMS_HISTORICAL_DB_HOST', default='127.0.0.1') +MYEMS_HISTORICAL_DB_PORT = config('MYEMS_HISTORICAL_DB_PORT', default=3306, cast=int) +MYEMS_HISTORICAL_DB_DATABASE = config('MYEMS_HISTORICAL_DB_DATABASE', default='myems_historical_db') +MYEMS_HISTORICAL_DB_USER = config('MYEMS_HISTORICAL_DB_USER', default='root') +MYEMS_HISTORICAL_DB_PASSWORD = config('MYEMS_HISTORICAL_DB_PASSWORD', default='!MyEMS1') + +INTERVAL_IN_SECONDS = config('INTERVAL_IN_SECONDS', default=600, cast=int) +GATEWAY_ID = config('GATEWAY_ID', cast=int) +GATEWAY_TOKEN = config('GATEWAY_TOKEN') myems_system_db = { - 'user': userID, - 'password': password, - 'host': '127.0.0.1', - 'database': 'myems_system_db', - 'port': 3306, + 'host': MYEMS_SYSTEM_DB_HOST, + 'port': MYEMS_SYSTEM_DB_PORT, + 'database': MYEMS_SYSTEM_DB_DATABASE, + 'user': MYEMS_SYSTEM_DB_USER, + 'password': MYEMS_SYSTEM_DB_PASSWORD, } myems_historical_db = { - 'user': userID, - 'password': password, - 'host': '127.0.0.1', - 'database': 'myems_historical_db', - 'port': 3306, + 'host': MYEMS_HISTORICAL_DB_HOST, + 'port': MYEMS_HISTORICAL_DB_PORT, + 'database': MYEMS_HISTORICAL_DB_DATABASE, + 'user': MYEMS_HISTORICAL_DB_USER, + 'password': MYEMS_HISTORICAL_DB_PASSWORD, } + # Indicates how long the process waits between readings -interval_in_seconds = 600 +interval_in_seconds = INTERVAL_IN_SECONDS # Get the gateway ID and token from MyEMS Admin # This is used for getting data sources associated with the gateway gateway = { - 'id': 1, - 'token': 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA' + 'id': GATEWAY_ID, + 'token': GATEWAY_TOKEN } diff --git a/myems-modbus-tcp/requirements.txt b/myems-modbus-tcp/requirements.txt index 784c235f..3f4c2ac6 100644 --- a/myems-modbus-tcp/requirements.txt +++ b/myems-modbus-tcp/requirements.txt @@ -1,5 +1,4 @@ -mysql-connector -mysql-connector-python==8.0.20 +mysql-connector-python modbus_tk schedule python-decouple diff --git a/myems-modbus-tcp/run.sh b/myems-modbus-tcp/run.sh old mode 100644 new mode 100755 diff --git a/myems-normalization/.env.example b/myems-normalization/.env.example new file mode 100644 index 00000000..6a0d24ca --- /dev/null +++ b/myems-normalization/.env.example @@ -0,0 +1,46 @@ +# config for myems_system_db +MYEMS_SYSTEM_DB_HOST=127.0.0.1 +MYEMS_SYSTEM_DB_PORT=3306 +MYEMS_SYSTEM_DB_DATABASE=myems_system_db +MYEMS_SYSTEM_DB_USER=root +MYEMS_SYSTEM_DB_PASSWORD=!MyEMS1 + +# config for myems_historical_db +MYEMS_HISTORICAL_DB_HOST=127.0.0.1 +MYEMS_HISTORICAL_DB_PORT=3306 +MYEMS_HISTORICAL_DB_DATABASE=myems_historical_db +MYEMS_HISTORICAL_DB_USER=root +MYEMS_HISTORICAL_DB_PASSWORD=!MyEMS1 + +# config for myems_energy_db +MYEMS_ENERGY_DB_HOST=127.0.0.1 +MYEMS_ENERGY_DB_PORT=3306 +MYEMS_ENERGY_DB_DATABASE=myems_energy_db +MYEMS_ENERGY_DB_USER=root +MYEMS_ENERGY_DB_PASSWORD=!MyEMS1 + +# config for myems_billing_db +MYEMS_BILLING_DB_HOST=127.0.0.1 +MYEMS_BILLING_DB_PORT=3306 +MYEMS_BILLING_DB_DATABASE=myems_billing_db +MYEMS_BILLING_DB_USER=root +MYEMS_BILLING_DB_PASSWORD=!MyEMS1 + +# indicates in how many minutes to normalize energy consumption +# 30 for half hourly +# 60 for hourly +MINUTES_TO_COUNT=60 + +# indicates within how many minutes to allow myems-cleaning service to clean the historical data +MINUTES_TO_CLEAN=30 + +# indicates from when (in UTC timezone) to calculate if the energy data is empty or were cleared +# format string: "%Y-%m-%d %H:%M:%S" +START_DATETIME_UTC="2019-12-31 16:00:00" + +# indicates the project's time zone offset from UTC +UTC_OFFSET=+08:00 + +# the number of worker processes in parallel for meter and virtual meter +# the pool size depends on the computing performance of the database server and the analysis server +POOL_SIZE=5 \ No newline at end of file diff --git a/myems-normalization/README.md b/myems-normalization/README.md index 00a3e4f7..ef2f4223 100644 --- a/myems-normalization/README.md +++ b/myems-normalization/README.md @@ -8,11 +8,22 @@ This service is a component of MyEMS and it normalizes energy data in historical ### Prerequisites -mysql.connector +mysql-connector-python + +openpyxl sympy -openpyxl +python-decouple + + +### Quick Run for Development + +```bash +pip install -r requirements.txt +chmod +x run.sh +run.sh +``` ### Installation @@ -22,7 +33,7 @@ cd ~/tools wget https://dev.mysql.com/get/Downloads/Connector-Python/mysql-connector-python-8.0.20.tar.gz tar xzf mysql-connector-python-8.0.20.tar.gz cd ~/tools/mysql-connector-python-8.0.20 -sudo python3 setup.py install +python3 setup.py install ``` Download and install mpmath @@ -30,7 +41,7 @@ Download and install mpmath cd ~/tools git clone https://github.com/fredrik-johansson/mpmath.git cd ~/tools/mpmath -sudo python3 setup.py install +python3 setup.py install ``` Download and install SymPy @@ -38,7 +49,7 @@ Download and install SymPy cd ~/tools git clone https://github.com/sympy/sympy.git cd ~/tools/sympy -sudo python3 setupegg.py develop +python3 setupegg.py develop ``` Download and install openpyxl @@ -62,11 +73,11 @@ tar xzf openpyxl-3.0.7.tar.gz ```bash cd ~/tools/et_xmlfile-1.1 -sudo python3 setup.py install +python3 setup.py install cd ~/tools/jdcal -sudo python3 setup.py install +python3 setup.py install cd ~/tools/openpyxl-3.0.7 -sudo python3 setup.py install +python3 setup.py install ``` Install myems-normalization service: @@ -74,29 +85,29 @@ Install myems-normalization service: cd ~ git clone https://github.com/MyEMS/myems.git cd myems -sudo git checkout master (or the latest release tag) -sudo cp -r ~/myems/myems-normalization /myems-normalization +git checkout master (or the latest release tag) +cp -r ~/myems/myems-normalization /myems-normalization ``` - -Edit config.py -``` -sudo nano /myems-normalization/config.py +Create .env file based on .env.example and edit the .env file if needed: +```bash +cp /myems-normalization/.env.example /myems-cleaning/.env +nano /myems-normalization/.env ``` Setup systemd service: ``` -sudo cp myems-normalization.service /lib/systemd/system/ +cp myems-normalization.service /lib/systemd/system/ ``` Enable the service: ``` -sudo systemctl enable myems-normalization.service +systemctl enable myems-normalization.service ``` Start the service: ``` -sudo systemctl start myems-normalization.service +systemctl start myems-normalization.service ``` Monitor the service: ```bash -sudo systemctl status myems-normalization.service +systemctl status myems-normalization.service ``` View the log: ```bash diff --git a/myems-normalization/config.py b/myems-normalization/config.py index 49413d77..b82709fa 100644 --- a/myems-normalization/config.py +++ b/myems-normalization/config.py @@ -1,49 +1,76 @@ from decouple import config -userID = config('userID', default='root') -password = config('password', default='') +MYEMS_SYSTEM_DB_HOST = config('MYEMS_SYSTEM_DB_HOST', default='127.0.0.1') +MYEMS_SYSTEM_DB_PORT = config('MYEMS_SYSTEM_DB_PORT', default=3306, cast=int) +MYEMS_SYSTEM_DB_DATABASE = config('MYEMS_SYSTEM_DB_DATABASE', default='myems_system_db') +MYEMS_SYSTEM_DB_USER = config('MYEMS_SYSTEM_DB_USER', default='root') +MYEMS_SYSTEM_DB_PASSWORD = config('MYEMS_SYSTEM_DB_PASSWORD', default='!MyEMS1') + +MYEMS_HISTORICAL_DB_HOST = config('MYEMS_HISTORICAL_DB_HOST', default='127.0.0.1') +MYEMS_HISTORICAL_DB_PORT = config('MYEMS_HISTORICAL_DB_PORT', default=3306, cast=int) +MYEMS_HISTORICAL_DB_DATABASE = config('MYEMS_HISTORICAL_DB_DATABASE', default='myems_system_db') +MYEMS_HISTORICAL_DB_USER = config('MYEMS_HISTORICAL_DB_USER', default='root') +MYEMS_HISTORICAL_DB_PASSWORD = config('MYEMS_HISTORICAL_DB_PASSWORD', default='!MyEMS1') + +MYEMS_ENERGY_DB_HOST = config('MYEMS_ENERGY_DB_HOST', default='127.0.0.1') +MYEMS_ENERGY_DB_PORT = config('MYEMS_ENERGY_DB_PORT', default=3306, cast=int) +MYEMS_ENERGY_DB_DATABASE = config('MYEMS_ENERGY_DB_DATABASE', default='myems_system_db') +MYEMS_ENERGY_DB_USER = config('MYEMS_ENERGY_DB_USER', default='root') +MYEMS_ENERGY_DB_PASSWORD = config('MYEMS_ENERGY_DB_PASSWORD', default='!MyEMS1') + +MYEMS_BILLING_DB_HOST = config('MYEMS_BILLING_DB_HOST', default='127.0.0.1') +MYEMS_BILLING_DB_PORT = config('MYEMS_BILLING_DB_PORT', default=3306, cast=int) +MYEMS_BILLING_DB_DATABASE = config('MYEMS_BILLING_DB_DATABASE', default='myems_system_db') +MYEMS_BILLING_DB_USER = config('MYEMS_BILLING_DB_USER', default='root') +MYEMS_BILLING_DB_PASSWORD = config('MYEMS_BILLING_DB_PASSWORD', default='!MyEMS1') + +MINUTES_TO_COUNT = config('MINUTES_TO_COUNT', default=60, cast=int) +MINUTES_TO_CLEAN = config('MINUTES_TO_CLEAN', default=30, cast=int) +START_DATETIME_UTC = config('START_DATETIME_UTC', default='2019-12-31 16:00:00') +UTC_OFFSET = config('UTC_OFFSET', default='+08:00') +POOL_SIZE = config('POOL_SIZE', default=5, cast=int) myems_system_db = { - 'user': userID, - 'password': password, - 'host': '127.0.0.1', - 'database': 'myems_system_db', - 'port': 3306, + 'host': MYEMS_SYSTEM_DB_HOST, + 'port': MYEMS_SYSTEM_DB_PORT, + 'database': MYEMS_SYSTEM_DB_DATABASE, + 'user': MYEMS_SYSTEM_DB_USER, + 'password': MYEMS_SYSTEM_DB_PASSWORD, } myems_energy_db = { - 'user': userID, - 'password': password, - 'host': '127.0.0.1', - 'database': 'myems_energy_db', - 'port': 3306, + 'host': MYEMS_ENERGY_DB_HOST, + 'port': MYEMS_ENERGY_DB_PORT, + 'database': MYEMS_ENERGY_DB_DATABASE, + 'user': MYEMS_ENERGY_DB_USER, + 'password': MYEMS_ENERGY_DB_PASSWORD, } myems_historical_db = { - 'user': userID, - 'password': password, - 'host': '127.0.0.1', - 'database': 'myems_historical_db', - 'port': 3306, + 'host': MYEMS_HISTORICAL_DB_HOST, + 'port': MYEMS_HISTORICAL_DB_PORT, + 'database': MYEMS_HISTORICAL_DB_DATABASE, + 'user': MYEMS_HISTORICAL_DB_USER, + 'password': MYEMS_HISTORICAL_DB_PASSWORD, } # indicates in how many minutes to normalize energy consumption # 30 for half hourly # 60 for hourly -minutes_to_count = 60 +minutes_to_count = MINUTES_TO_COUNT # indicates within how many minutes to allow myems-cleaning service to clean the historical data -minutes_to_clean = 30 +minutes_to_clean = MINUTES_TO_CLEAN # indicates from when (in UTC timezone) to calculate if the energy data is empty or were cleared -# format string: '%Y-%m-%d %H:%M:%S' -start_datetime_utc = '2019-12-31 16:00:00' +# format string: "%Y-%m-%d %H:%M:%S" +start_datetime_utc = START_DATETIME_UTC + +# indicates the project's time zone offset from UTC +utc_offset = UTC_OFFSET # the number of worker processes in parallel for meter and virtual meter # the pool size depends on the computing performance of the database server and the analysis server -pool_size = 5 - -# indicates the project's time zone offset from UTC -utc_offset = '+08:00' +pool_size = POOL_SIZE diff --git a/myems-normalization/requirements.txt b/myems-normalization/requirements.txt index 919ce2d2..f842dd19 100644 --- a/myems-normalization/requirements.txt +++ b/myems-normalization/requirements.txt @@ -1,5 +1,4 @@ -mysql-connector -mysql-connector-python==8.0.20 +mysql-connector-python openpyxl sympy python-decouple \ No newline at end of file