diff --git a/.gitignore b/.gitignore index 9a3a0b4a..31b9053f 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,4 @@ logs Thumbs.db # Folder config file -Desktop.ini \ No newline at end of file +Desktop.ini diff --git a/database/.gitignore b/database/.gitignore new file mode 100644 index 00000000..6f3370a5 --- /dev/null +++ b/database/.gitignore @@ -0,0 +1,45 @@ +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pythonenv* +# Logs +logs +*.log +*_build +*_static +*_templates +################# +## JetBrains +################# +.idea + +.vscode + +# System +.DS_Store + +# VIM swap files +.*.swp + +# VIM temp files +*~ + +################# +## Sublime Text +################# +*.sublime-* + +############ +## Windows +############ + +# Windows image file caches +Thumbs.db + +# Folder config file +Desktop.ini diff --git a/myems-aggregation/.env.example b/myems-aggregation/.env.example new file mode 100644 index 00000000..80de7464 --- /dev/null +++ b/myems-aggregation/.env.example @@ -0,0 +1,43 @@ +# config for myems_system_db +MYEMS_SYSTEM_DB_HOST=127.0.0.1 +MYEMS_SYSTEM_DB_PORT=3306 +MYEMS_SYSTEM_DB_DATABASE=myems_system_db +MYEMS_SYSTEM_DB_USER=root +MYEMS_SYSTEM_DB_PASSWORD=!MyEMS1 + +# config for myems_historical_db +MYEMS_HISTORICAL_DB_HOST=127.0.0.1 +MYEMS_HISTORICAL_DB_PORT=3306 +MYEMS_HISTORICAL_DB_DATABASE=myems_historical_db +MYEMS_HISTORICAL_DB_USER=root +MYEMS_HISTORICAL_DB_PASSWORD=!MyEMS1 + +# config for myems_energy_db +MYEMS_ENERGY_DB_HOST=127.0.0.1 +MYEMS_ENERGY_DB_PORT=3306 +MYEMS_ENERGY_DB_DATABASE=myems_energy_db +MYEMS_ENERGY_DB_USER=root +MYEMS_ENERGY_DB_PASSWORD=!MyEMS1 + +# config for myems_billing_db +MYEMS_BILLING_DB_HOST=127.0.0.1 +MYEMS_BILLING_DB_PORT=3306 +MYEMS_BILLING_DB_DATABASE=myems_billing_db +MYEMS_BILLING_DB_USER=root +MYEMS_BILLING_DB_PASSWORD=!MyEMS1 + +# indicates how long in minutes energy data will be aggregated +# 30 for half hourly +# 60 for hourly +MINUTES_TO_COUNT=60 + +# indicates from when (in UTC timezone) to recalculate if the energy data is null or were cleared +# format string: "%Y-%m-%d %H:%M:%S" +START_DATETIME_UTC="2019-12-31 16:00:00" + +# indicates the project's time zone offset from UTC +UTC_OFFSET=+08:00 + +# the number of worker processes in parallel +# the pool size depends on the computing performance of the database server and the analysis server +POOL_SIZE=5 \ No newline at end of file diff --git a/myems-aggregation/README.md b/myems-aggregation/README.md index ae0c6262..4a54a46d 100644 --- a/myems-aggregation/README.md +++ b/myems-aggregation/README.md @@ -6,7 +6,17 @@ This service is a component of MyEMS and it aggregates normalized data up to mul ### Prerequisites -mysql.connector +mysql-connector-python + +python-decouple + +### Quick Run for Development + +```bash +pip install -r requirements.txt +chmod +x run.sh +run.sh +``` ### Installation @@ -16,7 +26,7 @@ cd ~/tools wget https://dev.mysql.com/get/Downloads/Connector-Python/mysql-connector-python-8.0.20.tar.gz tar xzf mysql-connector-python-8.0.20.tar.gz cd ~/tools/mysql-connector-python-8.0.20 -sudo python3 setup.py install +python3 setup.py install ``` Install myems-aggregation service: @@ -24,28 +34,29 @@ Install myems-aggregation service: cd ~ git clone https://github.com/MyEMS/myems.git cd myems -sudo git checkout master (or the latest release tag) -sudo cp -R ~/myems/myems-aggregation /myems-aggregation +git checkout master (or the latest release tag) +cp -R ~/myems/myems-aggregation /myems-aggregation ``` -Edit config.py +Create .env file based on .env.example and edit the .env file if needed: ```bash -sudo nano /myems-aggregation/config.py +cp /myems-aggregation/.env.example /myems-aggregation/.env +nano /myems-aggregation/.env ``` Setup systemd service: ```bash -sudo cp myems-aggregation.service /lib/systemd/system/ +cp myems-aggregation.service /lib/systemd/system/ ``` Enable the service: ```bash -sudo systemctl enable myems-aggregation.service +systemctl enable myems-aggregation.service ``` Start the service: ```bash -sudo systemctl start myems-aggregation.service +systemctl start myems-aggregation.service ``` Monitor the service: ```bash -sudo systemctl status myems-aggregation.service +systemctl status myems-aggregation.service ``` View the log: ```bash diff --git a/myems-aggregation/config.py b/myems-aggregation/config.py index cddd91e1..3d63f55c 100644 --- a/myems-aggregation/config.py +++ b/myems-aggregation/config.py @@ -1,45 +1,79 @@ +from decouple import config + +MYEMS_SYSTEM_DB_HOST = config('MYEMS_SYSTEM_DB_HOST', default='127.0.0.1') +MYEMS_SYSTEM_DB_PORT = config('MYEMS_SYSTEM_DB_PORT', default=3306, cast=int) +MYEMS_SYSTEM_DB_DATABASE = config('MYEMS_SYSTEM_DB_DATABASE', default='myems_system_db') +MYEMS_SYSTEM_DB_USER = config('MYEMS_SYSTEM_DB_USER', default='root') +MYEMS_SYSTEM_DB_PASSWORD = config('MYEMS_SYSTEM_DB_PASSWORD', default='!MyEMS1') + +MYEMS_HISTORICAL_DB_HOST = config('MYEMS_HISTORICAL_DB_HOST', default='127.0.0.1') +MYEMS_HISTORICAL_DB_PORT = config('MYEMS_HISTORICAL_DB_PORT', default=3306, cast=int) +MYEMS_HISTORICAL_DB_DATABASE = config('MYEMS_HISTORICAL_DB_DATABASE', default='myems_system_db') +MYEMS_HISTORICAL_DB_USER = config('MYEMS_HISTORICAL_DB_USER', default='root') +MYEMS_HISTORICAL_DB_PASSWORD = config('MYEMS_HISTORICAL_DB_PASSWORD', default='!MyEMS1') + +MYEMS_ENERGY_DB_HOST = config('MYEMS_ENERGY_DB_HOST', default='127.0.0.1') +MYEMS_ENERGY_DB_PORT = config('MYEMS_ENERGY_DB_PORT', default=3306, cast=int) +MYEMS_ENERGY_DB_DATABASE = config('MYEMS_ENERGY_DB_DATABASE', default='myems_system_db') +MYEMS_ENERGY_DB_USER = config('MYEMS_ENERGY_DB_USER', default='root') +MYEMS_ENERGY_DB_PASSWORD = config('MYEMS_ENERGY_DB_PASSWORD', default='!MyEMS1') + +MYEMS_BILLING_DB_HOST = config('MYEMS_BILLING_DB_HOST', default='127.0.0.1') +MYEMS_BILLING_DB_PORT = config('MYEMS_BILLING_DB_PORT', default=3306, cast=int) +MYEMS_BILLING_DB_DATABASE = config('MYEMS_BILLING_DB_DATABASE', default='myems_system_db') +MYEMS_BILLING_DB_USER = config('MYEMS_BILLING_DB_USER', default='root') +MYEMS_BILLING_DB_PASSWORD = config('MYEMS_BILLING_DB_PASSWORD', default='!MyEMS1') + +MINUTES_TO_COUNT = config('MINUTES_TO_COUNT', default=60, cast=int) +START_DATETIME_UTC = config('START_DATETIME_UTC', default='2019-12-31 16:00:00') +UTC_OFFSET = config('UTC_OFFSET', default='+08:00') +POOL_SIZE = config('POOL_SIZE', default=5, cast=int) + myems_system_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_system_db', + 'host': MYEMS_SYSTEM_DB_HOST, + 'port': MYEMS_SYSTEM_DB_PORT, + 'database': MYEMS_SYSTEM_DB_DATABASE, + 'user': MYEMS_SYSTEM_DB_USER, + 'password': MYEMS_SYSTEM_DB_PASSWORD, } myems_historical_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_historical_db', + 'host': MYEMS_HISTORICAL_DB_HOST, + 'port': MYEMS_HISTORICAL_DB_PORT, + 'database': MYEMS_HISTORICAL_DB_DATABASE, + 'user': MYEMS_HISTORICAL_DB_USER, + 'password': MYEMS_HISTORICAL_DB_PASSWORD, } myems_energy_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_energy_db', + 'host': MYEMS_ENERGY_DB_HOST, + 'port': MYEMS_ENERGY_DB_PORT, + 'database': MYEMS_ENERGY_DB_DATABASE, + 'user': MYEMS_ENERGY_DB_USER, + 'password': MYEMS_ENERGY_DB_PASSWORD, } myems_billing_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_billing_db', + 'host': MYEMS_BILLING_DB_HOST, + 'port': MYEMS_BILLING_DB_PORT, + 'database': MYEMS_BILLING_DB_DATABASE, + 'user': MYEMS_BILLING_DB_USER, + 'password': MYEMS_BILLING_DB_PASSWORD, } - # indicates how long in minutes energy data will be aggregated # 30 for half hourly # 60 for hourly -minutes_to_count = 60 +minutes_to_count = MINUTES_TO_COUNT # indicates from when (in UTC timezone) to recalculate if the energy data is null or were cleared # format string: '%Y-%m-%d %H:%M:%S' -start_datetime_utc = '2019-12-31 16:00:00' +start_datetime_utc = START_DATETIME_UTC # indicates the project's time zone offset from UTC -utc_offset = '+08:00' +utc_offset = UTC_OFFSET # the number of worker processes in parallel # the pool size depends on the computing performance of the database server and the analysis server -pool_size = 5 +pool_size = POOL_SIZE diff --git a/myems-aggregation/requirements.txt b/myems-aggregation/requirements.txt index a6dc3627..c4beab52 100644 --- a/myems-aggregation/requirements.txt +++ b/myems-aggregation/requirements.txt @@ -1 +1,2 @@ -mysql-connector \ No newline at end of file +mysql-connector-python +python-decouple \ No newline at end of file diff --git a/myems-aggregation/run.sh b/myems-aggregation/run.sh new file mode 100755 index 00000000..cbc5f14c --- /dev/null +++ b/myems-aggregation/run.sh @@ -0,0 +1 @@ +python3 main.py \ No newline at end of file diff --git a/myems-api/.env.example b/myems-api/.env.example new file mode 100644 index 00000000..f4c2db4a --- /dev/null +++ b/myems-api/.env.example @@ -0,0 +1,82 @@ +# config for myems_system_db +MYEMS_SYSTEM_DB_HOST=127.0.0.1 +MYEMS_SYSTEM_DB_PORT=3306 +MYEMS_SYSTEM_DB_DATABASE=myems_system_db +MYEMS_SYSTEM_DB_USER=root +MYEMS_SYSTEM_DB_PASSWORD=!MyEMS1 + +# config for myems_energy_db +MYEMS_ENERGY_DB_HOST=127.0.0.1 +MYEMS_ENERGY_DB_PORT=3306 +MYEMS_ENERGY_DB_DATABASE=myems_energy_db +MYEMS_ENERGY_DB_USER=root +MYEMS_ENERGY_DB_PASSWORD=!MyEMS1 + +# config for myems_energy_baseline_db +MYEMS_ENERGY_BASELINE_DB_HOST=127.0.0.1 +MYEMS_ENERGY_BASELINE_DB_PORT=3306 +MYEMS_ENERGY_BASELINE_DB_DATABASE=myems_energy_baseline_db +MYEMS_ENERGY_BASELINE_DB_USER=root +MYEMS_ENERGY_BASELINE_DB_PASSWORD=!MyEMS1 + +# config for myems_billing_db +MYEMS_BILLING_DB_HOST=127.0.0.1 +MYEMS_BILLING_DB_PORT=3306 +MYEMS_BILLING_DB_DATABASE=myems_billing_db +MYEMS_BILLING_DB_USER=root +MYEMS_BILLING_DB_PASSWORD=!MyEMS1 + +# config for myems_billing_baseline_db +MYEMS_BILLING_BASELINE_DB_HOST=127.0.0.1 +MYEMS_BILLING_BASELINE_DB_PORT=3306 +MYEMS_BILLING_BASELINE_DB_DATABASE=myems_billing_baseline_db +MYEMS_BILLING_BASELINE_DB_USER=root +MYEMS_BILLING_BASELINE_DB_PASSWORD=!MyEMS1 + +# config for myems_historical_db +MYEMS_HISTORICAL_DB_HOST=127.0.0.1 +MYEMS_HISTORICAL_DB_PORT=3306 +MYEMS_HISTORICAL_DB_DATABASE=myems_historical_db +MYEMS_HISTORICAL_DB_USER=root +MYEMS_HISTORICAL_DB_PASSWORD=!MyEMS1 + +# config for myems_user_db +MYEMS_USER_DB_HOST=127.0.0.1 +MYEMS_USER_DB_PORT=3306 +MYEMS_USER_DB_DATABASE=myems_user_db +MYEMS_USER_DB_USER=root +MYEMS_USER_DB_PASSWORD=!MyEMS1 + +# config for myems_fdd_db +MYEMS_FDD_DB_HOST=127.0.0.1 +MYEMS_FDD_DB_PORT=3306 +MYEMS_FDD_DB_DATABASE=myems_fdd_db +MYEMS_FDD_DB_USER=root +MYEMS_FDD_DB_PASSWORD=!MyEMS1 + +# config for myems_reporting_db +MYEMS_REPORTING_DB_HOST=127.0.0.1 +MYEMS_REPORTING_DB_PORT=3306 +MYEMS_REPORTING_DB_DATABASE=myems_reporting_db +MYEMS_REPORTING_DB_USER=root +MYEMS_REPORTING_DB_PASSWORD=!MyEMS1 + +# indicated in how many minutes to calculate meter energy consumption +# 30 for half hourly period +# 60 for hourly period +MINUTES_TO_COUNT=60 + +# indicates the project's time zone offset from UTC +UTC_OFFSET=+08:00 + +# indicates from when ( in local timezone) of the day to calculate working days +WORKING_DAY_START_TIME_LOCAL=00:00:00 + +# indicates where user uploaded files will be saved to +# must use the root folder of myems-admin web application +# for example if you serve myems-admin at /var/www/html/admin +# you should set the upload_path as below +UPLOAD_PATH=/var/www/html/admin/upload/ + +# main currency unit +CURRENCY_UNIT=CNY \ No newline at end of file diff --git a/myems-api/.gitignore b/myems-api/.gitignore index 83d71500..4b3fcb2b 100644 --- a/myems-api/.gitignore +++ b/myems-api/.gitignore @@ -50,3 +50,13 @@ doc/_build *~ advancereport/ + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pythonenv* diff --git a/myems-api/README.md b/myems-api/README.md index bedde402..3e3be42b 100644 --- a/myems-api/README.md +++ b/myems-api/README.md @@ -19,6 +19,13 @@ gunicorn openpyxl +## Quick Run for Development + +```bash +pip install -r requirements.txt +chmod +x run.sh +run.sh +``` ## Installation @@ -31,7 +38,7 @@ For macOS developers, please refer to [Installation on macOS (Chinese)](./instal cd ~/tools git clone https://github.com/c0fec0de/anytree.git cd anytree -sudo python3 setup.py install +python3 setup.py install ``` * Install simplejson @@ -39,7 +46,7 @@ sudo python3 setup.py install cd ~/tools git clone https://github.com/simplejson/simplejson.git cd simplejson -sudo python3 setup.py install +python3 setup.py install ``` * Install MySQL Connector @@ -48,7 +55,7 @@ sudo python3 setup.py install wget https://dev.mysql.com/get/Downloads/Connector-Python/mysql-connector-python-8.0.20.tar.gz tar xzf mysql-connector-python-8.0.20.tar.gz cd ~/tools/mysql-connector-python-8.0.20 - sudo python3 setup.py install + python3 setup.py install ``` * Install Falcon, @@ -67,15 +74,15 @@ sudo python3 setup.py install pip3 download cython falcon falcon-cors falcon-multipart export LC_ALL="en_US.UTF-8" export LC_CTYPE="en_US.UTF-8" - sudo dpkg-reconfigure locales - sudo pip3 install --upgrade --no-index --find-links ~/tools/falcon cython falcon falcon-cors falcon-multipart + dpkg-reconfigure locales + pip3 install --upgrade --no-index --find-links ~/tools/falcon cython falcon falcon-cors falcon-multipart ``` * Install gunicorn, refer to http://gunicorn.org ```bash mkdir ~/tools/gunicorn && cd ~/tools/gunicorn pip3 download gunicorn - sudo pip3 install --no-index --find-links ~/tools/gunicorn gunicorn + pip3 install --no-index --find-links ~/tools/gunicorn gunicorn ``` * Install openpyxl, refer to https://foss.heptapod.net/openpyxl/openpyxl @@ -104,12 +111,12 @@ tar xzf openpyxl-3.0.7.tar.gz ```bash cd ~/tools/et_xmlfile-1.1 -sudo python3 setup.py install +python3 setup.py install cd ~/tools/jdcal -sudo python3 setup.py install -sudo pip3 install --no-index --find-links ~/tools/pillow Pillow +python3 setup.py install +pip3 install --no-index --find-links ~/tools/pillow Pillow cd ~/tools/openpyxl-3.0.7 -sudo python3 setup.py install +python3 setup.py install ``` * Download myems: @@ -121,43 +128,38 @@ git clone https://github.com/MyEMS/myems.git * Install gunicorn service for myems-api: ```bash cd ~/myems/myems-api -sudo cp -R ~/myems/myems-api /myems-api +cp -R ~/myems/myems-api /myems-api ``` - Change IP address in the config file: +Create .env file based on .env.example and edit the .env file if needed: ```bash -sudo nano /myems-api/config.py +cp /myems-api/.env.example /myems-api/.env +nano /myems-api/.env ``` - Change the listening port (default is 8000) in gunicorn.socket: +Change the listening port (default is 8000) in gunicorn.socket: ```bash -sudo nano /myems-api/myems-api.socket +nano /myems-api/myems-api.socket ``` ```bash ListenStream=0.0.0.0:8000 ``` ```bash -sudo ufw allow 8000 +ufw allow 8000 ``` - Setup systemd configure files: +Setup systemd configure files: ```bash - sudo cp /myems-api/myems-api.service /lib/systemd/system/ - sudo cp /myems-api/myems-api.socket /lib/systemd/system/ - sudo cp /myems-api/myems-api.conf /usr/lib/tmpfiles.d/ +cp /myems-api/myems-api.service /lib/systemd/system/ +cp /myems-api/myems-api.socket /lib/systemd/system/ +cp /myems-api/myems-api.conf /usr/lib/tmpfiles.d/ ``` Next enable the services so that they autostart at boot: ```bash - sudo systemctl enable myems-api.socket - sudo systemctl enable myems-api.service + systemctl enable myems-api.socket + systemctl enable myems-api.service ``` - Start the services : +Start the services : ```bash - sudo systemctl start myems-api.socket - sudo systemctl start myems-api.service -``` - -## Run for debugging and testing -```bash -cd myems-api -sudo gunicorn -b 127.0.0.1:8000 app:api +systemctl start myems-api.socket +systemctl start myems-api.service ``` ## API List diff --git a/myems-api/config.py b/myems-api/config.py index b762986a..29b019ae 100644 --- a/myems-api/config.py +++ b/myems-api/config.py @@ -1,82 +1,154 @@ +from decouple import config + + +MYEMS_SYSTEM_DB_HOST = config('MYEMS_SYSTEM_DB_HOST', default='127.0.0.1') +MYEMS_SYSTEM_DB_PORT = config('MYEMS_SYSTEM_DB_PORT', default=3306, cast=int) +MYEMS_SYSTEM_DB_DATABASE = config('MYEMS_SYSTEM_DB_DATABASE', default='myems_system_db') +MYEMS_SYSTEM_DB_USER = config('MYEMS_SYSTEM_DB_USER', default='root') +MYEMS_SYSTEM_DB_PASSWORD = config('MYEMS_SYSTEM_DB_PASSWORD', default='!MyEMS1') + +MYEMS_ENERGY_DB_HOST = config('MYEMS_ENERGY_DB_HOST', default='127.0.0.1') +MYEMS_ENERGY_DB_PORT = config('MYEMS_ENERGY_DB_PORT', default=3306, cast=int) +MYEMS_ENERGY_DB_DATABASE = config('MYEMS_ENERGY_DB_DATABASE', default='myems_energy_db') +MYEMS_ENERGY_DB_USER = config('MYEMS_ENERGY_DB_USER', default='root') +MYEMS_ENERGY_DB_PASSWORD = config('MYEMS_ENERGY_DB_PASSWORD', default='!MyEMS1') + +MYEMS_ENERGY_BASELINE_DB_HOST = config('MYEMS_ENERGY_BASELINE_DB_HOST', default='127.0.0.1') +MYEMS_ENERGY_BASELINE_DB_PORT = config('MYEMS_ENERGY_BASELINE_DB_PORT', default=3306, cast=int) +MYEMS_ENERGY_BASELINE_DB_DATABASE = config('MYEMS_ENERGY_BASELINE_DB_DATABASE', default='myems_energy_baseline_db') +MYEMS_ENERGY_BASELINE_DB_USER = config('MYEMS_ENERGY_BASELINE_DB_USER', default='root') +MYEMS_ENERGY_BASELINE_DB_PASSWORD = config('MYEMS_ENERGY_BASELINE_DB_PASSWORD', default='!MyEMS1') + +MYEMS_BILLING_DB_HOST = config('MYEMS_BILLING_DB_HOST', default='127.0.0.1') +MYEMS_BILLING_DB_PORT = config('MYEMS_BILLING_DB_PORT', default=3306, cast=int) +MYEMS_BILLING_DB_DATABASE = config('MYEMS_BILLING_DB_DATABASE', default='myems_billing_db') +MYEMS_BILLING_DB_USER = config('MYEMS_BILLING_DB_USER', default='root') +MYEMS_BILLING_DB_PASSWORD = config('MYEMS_BILLING_DB_PASSWORD', default='!MyEMS1') + +MYEMS_BILLING_BASELINE_DB_HOST = config('MYEMS_BILLING_BASELINE_DB_HOST', default='127.0.0.1') +MYEMS_BILLING_BASELINE_DB_PORT = config('MYEMS_BILLING_BASELINE_DB_PORT', default=3306, cast=int) +MYEMS_BILLING_BASELINE_DB_DATABASE = config('MYEMS_BILLING_BASELINE_DB_DATABASE', default='myems_billing_baseline_db') +MYEMS_BILLING_BASELINE_DB_USER = config('MYEMS_BILLING_BASELINE_DB_USER', default='root') +MYEMS_BILLING_BASELINE_DB_PASSWORD = config('MYEMS_BILLING_BASELINE_DB_PASSWORD', default='!MyEMS1') + +MYEMS_HISTORICAL_DB_HOST = config('MYEMS_HISTORICAL_DB_HOST', default='127.0.0.1') +MYEMS_HISTORICAL_DB_PORT = config('MYEMS_HISTORICAL_DB_PORT', default=3306, cast=int) +MYEMS_HISTORICAL_DB_DATABASE = config('MYEMS_HISTORICAL_DB_DATABASE', default='myems_historical_db') +MYEMS_HISTORICAL_DB_USER = config('MYEMS_HISTORICAL_DB_USER', default='root') +MYEMS_HISTORICAL_DB_PASSWORD = config('MYEMS_HISTORICAL_DB_PASSWORD', default='!MyEMS1') + +MYEMS_USER_DB_HOST = config('MYEMS_USER_DB_HOST', default='127.0.0.1') +MYEMS_USER_DB_PORT = config('MYEMS_USER_DB_PORT', default=3306, cast=int) +MYEMS_USER_DB_DATABASE = config('MYEMS_USER_DB_DATABASE', default='myems_user_db') +MYEMS_USER_DB_USER = config('MYEMS_USER_DB_USER', default='root') +MYEMS_USER_DB_PASSWORD = config('MYEMS_USER_DB_PASSWORD', default='!MyEMS1') + +MYEMS_FDD_DB_HOST = config('MYEMS_FDD_DB_HOST', default='127.0.0.1') +MYEMS_FDD_DB_PORT = config('MYEMS_FDD_DB_PORT', default=3306, cast=int) +MYEMS_FDD_DB_DATABASE = config('MYEMS_FDD_DB_DATABASE', default='myems_fdd_db') +MYEMS_FDD_DB_USER = config('MYEMS_FDD_DB_USER', default='root') +MYEMS_FDD_DB_PASSWORD = config('MYEMS_FDD_DB_PASSWORD', default='!MyEMS1') + +MYEMS_REPORTING_DB_HOST = config('MYEMS_REPORTING_DB_HOST', default='127.0.0.1') +MYEMS_REPORTING_DB_PORT = config('MYEMS_REPORTING_DB_PORT', default=3306, cast=int) +MYEMS_REPORTING_DB_DATABASE = config('MYEMS_REPORTING_DB_DATABASE', default='myems_reporting_db') +MYEMS_REPORTING_DB_USER = config('MYEMS_REPORTING_DB_USER', default='root') +MYEMS_REPORTING_DB_PASSWORD = config('MYEMS_REPORTING_DB_PASSWORD', default='!MyEMS1') + +MINUTES_TO_COUNT = config('MINUTES_TO_COUNT', default=60, cast=int) +UTC_OFFSET = config('UTC_OFFSET', default='+08:00') +WORKING_DAY_START_TIME_LOCAL = config('WORKING_DAY_START_TIME_LOCAL', default='00:00:00') +UPLOAD_PATH = config('UPLOAD_PATH', default='/var/www/html/admin/upload/') +CURRENCY_UNIT = config('CURRENCY_UNIT', default='CNY') + myems_system_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_system_db', + 'host': MYEMS_SYSTEM_DB_HOST, + 'port': MYEMS_SYSTEM_DB_PORT, + 'database': MYEMS_SYSTEM_DB_DATABASE, + 'user': MYEMS_SYSTEM_DB_USER, + 'password': MYEMS_SYSTEM_DB_PASSWORD, } myems_energy_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_energy_db', + 'host': MYEMS_ENERGY_DB_HOST, + 'port': MYEMS_ENERGY_DB_PORT, + 'database': MYEMS_ENERGY_DB_DATABASE, + 'user': MYEMS_ENERGY_DB_USER, + 'password': MYEMS_ENERGY_DB_PASSWORD, } myems_energy_baseline_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_energy_baseline_db', + 'host': MYEMS_ENERGY_BASELINE_DB_HOST, + 'port': MYEMS_ENERGY_BASELINE_DB_PORT, + 'database': MYEMS_ENERGY_BASELINE_DB_DATABASE, + 'user': MYEMS_ENERGY_BASELINE_DB_USER, + 'password': MYEMS_ENERGY_BASELINE_DB_PASSWORD, } myems_billing_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_billing_db', + 'host': MYEMS_BILLING_DB_HOST, + 'port': MYEMS_BILLING_DB_PORT, + 'database': MYEMS_BILLING_DB_DATABASE, + 'user': MYEMS_BILLING_DB_USER, + 'password': MYEMS_BILLING_DB_PASSWORD, } myems_billing_baseline_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_billing_baseline_db', + 'host': MYEMS_BILLING_BASELINE_DB_HOST, + 'port': MYEMS_BILLING_BASELINE_DB_PORT, + 'database': MYEMS_BILLING_BASELINE_DB_DATABASE, + 'user': MYEMS_BILLING_BASELINE_DB_USER, + 'password': MYEMS_BILLING_BASELINE_DB_PASSWORD, } myems_historical_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_historical_db', + 'host': MYEMS_HISTORICAL_DB_HOST, + 'port': MYEMS_HISTORICAL_DB_PORT, + 'database': MYEMS_HISTORICAL_DB_DATABASE, + 'user': MYEMS_HISTORICAL_DB_USER, + 'password': MYEMS_HISTORICAL_DB_PASSWORD, } myems_user_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_user_db', + 'host': MYEMS_USER_DB_HOST, + 'port': MYEMS_USER_DB_PORT, + 'database': MYEMS_USER_DB_DATABASE, + 'user': MYEMS_USER_DB_USER, + 'password': MYEMS_USER_DB_PASSWORD, } myems_fdd_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_fdd_db', + 'host': MYEMS_FDD_DB_HOST, + 'port': MYEMS_FDD_DB_PORT, + 'database': MYEMS_FDD_DB_DATABASE, + 'user': MYEMS_FDD_DB_USER, + 'password': MYEMS_FDD_DB_PASSWORD, } myems_reporting_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_reporting_db', + 'host': MYEMS_REPORTING_DB_HOST, + 'port': MYEMS_REPORTING_DB_PORT, + 'database': MYEMS_REPORTING_DB_DATABASE, + 'user': MYEMS_REPORTING_DB_USER, + 'password': MYEMS_REPORTING_DB_PASSWORD, } # indicated in how many minutes to calculate meter energy consumption # 30 for half hourly period # 60 for hourly period -minutes_to_count = 60 +minutes_to_count = MINUTES_TO_COUNT # indicates the project's time zone offset from UTC -utc_offset = '+08:00' +utc_offset = UTC_OFFSET # indicates from when ( in local timezone) of the day to calculate working days -working_day_start_time_local = '00:00:00' +working_day_start_time_local = WORKING_DAY_START_TIME_LOCAL # indicates where user uploaded files will be saved to # must use the root folder of myems-admin web application # for example if you serve myems-admin at /var/www/html/admin # you should set the upload_path as below -upload_path = '/var/www/html/admin/upload/' +upload_path = UPLOAD_PATH # main currency unit -currency_unit = 'CNY' +currency_unit = CURRENCY_UNIT diff --git a/myems-api/requirements.txt b/myems-api/requirements.txt index 12bd96f0..b2db7617 100644 --- a/myems-api/requirements.txt +++ b/myems-api/requirements.txt @@ -1,9 +1,12 @@ anytree simplejson -mysql.connector +mysql-connector-python falcon falcon_cors falcon-multipart gunicorn +et_xmlfile +jdcal openpyxl -pillow \ No newline at end of file +pillow +python-decouple \ No newline at end of file diff --git a/myems-api/run.sh b/myems-api/run.sh new file mode 100755 index 00000000..e4aa9d3f --- /dev/null +++ b/myems-api/run.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +gunicorn --pid pid --timeout 600 --workers=4 app:api \ No newline at end of file diff --git a/myems-cleaning/.env.example b/myems-cleaning/.env.example new file mode 100644 index 00000000..cd639b32 --- /dev/null +++ b/myems-cleaning/.env.example @@ -0,0 +1,21 @@ +# config for myems_system_db +MYEMS_SYSTEM_DB_HOST=127.0.0.1 +MYEMS_SYSTEM_DB_PORT=3306 +MYEMS_SYSTEM_DB_DATABASE=myems_system_db +MYEMS_SYSTEM_DB_USER=root +MYEMS_SYSTEM_DB_PASSWORD=!MyEMS1 + +# config for myems_historical_db +MYEMS_HISTORICAL_DB_HOST=127.0.0.1 +MYEMS_HISTORICAL_DB_PORT=3306 +MYEMS_HISTORICAL_DB_DATABASE=myems_historical_db +MYEMS_HISTORICAL_DB_USER=root +MYEMS_HISTORICAL_DB_PASSWORD=!MyEMS1 + +# indicates how long analog values and digital values will be kept in database +# the longer days the more memory and disc space needed. +# NOTE: By default, energy values in historical db will never be deleted automatically. +LIVE_IN_DAYS=365 + +# indicates if the program is in debug mode +IS_DEBUG=False \ No newline at end of file diff --git a/myems-cleaning/README.md b/myems-cleaning/README.md index 0ac571ab..605628eb 100644 --- a/myems-cleaning/README.md +++ b/myems-cleaning/README.md @@ -9,7 +9,18 @@ This service is a component of MyEMS and it cleans the historical data. ### Prerequisites -mysql.connector +mysql-connector-python + +schedule + +python-decouple + +### Quick Run for Development +```bash +pip install -r requirements.txt +chmod +x run.sh +run.sh +``` ### Installation @@ -19,7 +30,7 @@ cd ~/tools wget https://dev.mysql.com/get/Downloads/Connector-Python/mysql-connector-python-8.0.20.tar.gz tar xzf mysql-connector-python-8.0.20.tar.gz cd ~/tools/mysql-connector-python-8.0.20 -sudo python3 setup.py install +python3 setup.py install ``` Install myems-cleaning service @@ -27,28 +38,29 @@ Install myems-cleaning service cd ~ git clone https://github.com/MyEMS/myems.git cd myems -sudo git checkout master (or the latest release tag) -sudo cp -R ~/myems/myems-cleaning /myems-cleaning +git checkout master (or the latest release tag) +cp -R ~/myems/myems-cleaning /myems-cleaning ``` -Open config file and edit database configuration +Create .env file based on .env.example and edit the .env file if needed: ```bash -sudo nano /myems-cleaning/config.py +cp /myems-cleaning/.env.example /myems-cleaning/.env +nano /myems-cleaning/.env ``` Setup systemd service: ```bash -sudo cp myems-cleaning.service /lib/systemd/system/ +cp myems-cleaning.service /lib/systemd/system/ ``` Enable the service: ```bash -sudo systemctl enable myems-cleaning.service +systemctl enable myems-cleaning.service ``` Start the service: ```bash -sudo systemctl start myems-cleaning.service +systemctl start myems-cleaning.service ``` Monitor the service: ```bash -sudo systemctl status myems-cleaning.service +systemctl status myems-cleaning.service ``` View the log: ```bash diff --git a/myems-cleaning/config.py b/myems-cleaning/config.py index b2eb0384..dbaabef0 100644 --- a/myems-cleaning/config.py +++ b/myems-cleaning/config.py @@ -1,24 +1,41 @@ +from decouple import config + + +MYEMS_SYSTEM_DB_HOST = config('MYEMS_SYSTEM_DB_HOST', default='127.0.0.1') +MYEMS_SYSTEM_DB_PORT = config('MYEMS_SYSTEM_DB_PORT', default=3306, cast=int) +MYEMS_SYSTEM_DB_DATABASE = config('MYEMS_SYSTEM_DB_DATABASE', default='myems_system_db') +MYEMS_SYSTEM_DB_USER = config('MYEMS_SYSTEM_DB_USER', default='root') +MYEMS_SYSTEM_DB_PASSWORD = config('MYEMS_SYSTEM_DB_PASSWORD', default='!MyEMS1') + +MYEMS_HISTORICAL_DB_HOST = config('MYEMS_HISTORICAL_DB_HOST', default='127.0.0.1') +MYEMS_HISTORICAL_DB_PORT = config('MYEMS_HISTORICAL_DB_PORT', default=3306, cast=int) +MYEMS_HISTORICAL_DB_DATABASE = config('MYEMS_HISTORICAL_DB_DATABASE', default='myems_historical_db') +MYEMS_HISTORICAL_DB_USER = config('MYEMS_HISTORICAL_DB_USER', default='root') +MYEMS_HISTORICAL_DB_PASSWORD = config('MYEMS_HISTORICAL_DB_PASSWORD', default='!MyEMS1') + +LIVE_IN_DAYS = config('LIVE_IN_DAYS', default=365, cast=int) +IS_DEBUG = config('IS_DEBUG', default=False, cast=bool) + myems_system_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_system_db', - 'port': 3306, + 'host': MYEMS_SYSTEM_DB_HOST, + 'port': MYEMS_SYSTEM_DB_PORT, + 'database': MYEMS_SYSTEM_DB_DATABASE, + 'user': MYEMS_SYSTEM_DB_USER, + 'password': MYEMS_SYSTEM_DB_PASSWORD, } myems_historical_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_historical_db', - 'port': 3306, + 'host': MYEMS_HISTORICAL_DB_HOST, + 'port': MYEMS_HISTORICAL_DB_PORT, + 'database': MYEMS_HISTORICAL_DB_DATABASE, + 'user': MYEMS_HISTORICAL_DB_USER, + 'password': MYEMS_HISTORICAL_DB_PASSWORD, } # indicates how long analog values and digital values will be kept in database # the longer days the more memory and disc space needed. -live_in_days = 365 - # NOTE: By default, energy values in historical db will never be deleted automatically. +live_in_days = LIVE_IN_DAYS # indicates if the program is in debug mode -is_debug = False +is_debug = IS_DEBUG diff --git a/myems-cleaning/requirements.txt b/myems-cleaning/requirements.txt index 51fbac7f..7ba59426 100644 --- a/myems-cleaning/requirements.txt +++ b/myems-cleaning/requirements.txt @@ -1,2 +1,3 @@ -mysql-connector -schedule \ No newline at end of file +mysql-connector-python +schedule +python-decouple \ No newline at end of file diff --git a/myems-cleaning/run.sh b/myems-cleaning/run.sh new file mode 100644 index 00000000..7a84f979 --- /dev/null +++ b/myems-cleaning/run.sh @@ -0,0 +1 @@ +python main.py \ No newline at end of file diff --git a/myems-modbus-tcp/.env.example b/myems-modbus-tcp/.env.example new file mode 100644 index 00000000..8fc9a034 --- /dev/null +++ b/myems-modbus-tcp/.env.example @@ -0,0 +1,21 @@ +# config for myems_system_db +MYEMS_SYSTEM_DB_HOST=127.0.0.1 +MYEMS_SYSTEM_DB_PORT=3306 +MYEMS_SYSTEM_DB_DATABASE=myems_system_db +MYEMS_SYSTEM_DB_USER=root +MYEMS_SYSTEM_DB_PASSWORD=!MyEMS1 + +# config for myems_historical_db +MYEMS_HISTORICAL_DB_HOST=127.0.0.1 +MYEMS_HISTORICAL_DB_PORT=3306 +MYEMS_HISTORICAL_DB_DATABASE=myems_historical_db +MYEMS_HISTORICAL_DB_USER=root +MYEMS_HISTORICAL_DB_PASSWORD=!MyEMS1 + +# Indicates how long the process waits between readings +INTERVAL_IN_SECONDS=600 + +# Get the gateway ID and token from MyEMS Admin +# This is used for getting data sources associated with the gateway +GATEWAY_ID=1 +GATEWAY_TOKEN=AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA \ No newline at end of file diff --git a/myems-modbus-tcp/README.md b/myems-modbus-tcp/README.md index 088e5c65..1e0acc63 100644 --- a/myems-modbus-tcp/README.md +++ b/myems-modbus-tcp/README.md @@ -4,13 +4,23 @@ This service is a component of MyEMS to acquire data from Modbus TCP devices. ### Prerequisites -pyserial -modbus-tk +mysql-connector-python -mysql.connector +modbus_tk -Schedule +schedule + +python-decouple + + +### Quick Run for Development + +```bash +pip install -r requirements.txt +chmod +x run.sh +run.sh +``` ### Installation @@ -50,9 +60,10 @@ cd myems git checkout master (or the latest release tag) cp -R ~/myems/myems-modbus-tcp /myems-modbus-tcp ``` -Edit the config +Create .env file based on .env.example and edit the .env file if needed: ```bash -nano /myems-modbus-tcp/config.py +cp /myems-modbus-tcp/.env.example /myems-cleaning/.env +nano /myems-modbus-tcp/.env ``` Setup systemd service: ```bash diff --git a/myems-modbus-tcp/config.py b/myems-modbus-tcp/config.py index 212aaf51..bfe85e2f 100644 --- a/myems-modbus-tcp/config.py +++ b/myems-modbus-tcp/config.py @@ -1,25 +1,45 @@ +from decouple import config + + +MYEMS_SYSTEM_DB_HOST = config('MYEMS_SYSTEM_DB_HOST', default='127.0.0.1') +MYEMS_SYSTEM_DB_PORT = config('MYEMS_SYSTEM_DB_PORT', default=3306, cast=int) +MYEMS_SYSTEM_DB_DATABASE = config('MYEMS_SYSTEM_DB_DATABASE', default='myems_system_db') +MYEMS_SYSTEM_DB_USER = config('MYEMS_SYSTEM_DB_USER', default='root') +MYEMS_SYSTEM_DB_PASSWORD = config('MYEMS_SYSTEM_DB_PASSWORD', default='!MyEMS1') + +MYEMS_HISTORICAL_DB_HOST = config('MYEMS_HISTORICAL_DB_HOST', default='127.0.0.1') +MYEMS_HISTORICAL_DB_PORT = config('MYEMS_HISTORICAL_DB_PORT', default=3306, cast=int) +MYEMS_HISTORICAL_DB_DATABASE = config('MYEMS_HISTORICAL_DB_DATABASE', default='myems_historical_db') +MYEMS_HISTORICAL_DB_USER = config('MYEMS_HISTORICAL_DB_USER', default='root') +MYEMS_HISTORICAL_DB_PASSWORD = config('MYEMS_HISTORICAL_DB_PASSWORD', default='!MyEMS1') + +INTERVAL_IN_SECONDS = config('INTERVAL_IN_SECONDS', default=600, cast=int) +GATEWAY_ID = config('GATEWAY_ID', cast=int) +GATEWAY_TOKEN = config('GATEWAY_TOKEN') + myems_system_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_system_db', - 'port': 3306, + 'host': MYEMS_SYSTEM_DB_HOST, + 'port': MYEMS_SYSTEM_DB_PORT, + 'database': MYEMS_SYSTEM_DB_DATABASE, + 'user': MYEMS_SYSTEM_DB_USER, + 'password': MYEMS_SYSTEM_DB_PASSWORD, } myems_historical_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_historical_db', - 'port': 3306, + 'host': MYEMS_HISTORICAL_DB_HOST, + 'port': MYEMS_HISTORICAL_DB_PORT, + 'database': MYEMS_HISTORICAL_DB_DATABASE, + 'user': MYEMS_HISTORICAL_DB_USER, + 'password': MYEMS_HISTORICAL_DB_PASSWORD, } + # Indicates how long the process waits between readings -interval_in_seconds = 600 +interval_in_seconds = INTERVAL_IN_SECONDS # Get the gateway ID and token from MyEMS Admin # This is used for getting data sources associated with the gateway gateway = { - 'id': 1, - 'token': 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA' + 'id': GATEWAY_ID, + 'token': GATEWAY_TOKEN } diff --git a/myems-modbus-tcp/requirements.txt b/myems-modbus-tcp/requirements.txt index a867d987..3f4c2ac6 100644 --- a/myems-modbus-tcp/requirements.txt +++ b/myems-modbus-tcp/requirements.txt @@ -1,3 +1,4 @@ -mysql-connector +mysql-connector-python modbus_tk schedule +python-decouple diff --git a/myems-modbus-tcp/run.sh b/myems-modbus-tcp/run.sh new file mode 100755 index 00000000..7a84f979 --- /dev/null +++ b/myems-modbus-tcp/run.sh @@ -0,0 +1 @@ +python main.py \ No newline at end of file diff --git a/myems-normalization/.env.example b/myems-normalization/.env.example new file mode 100644 index 00000000..6a0d24ca --- /dev/null +++ b/myems-normalization/.env.example @@ -0,0 +1,46 @@ +# config for myems_system_db +MYEMS_SYSTEM_DB_HOST=127.0.0.1 +MYEMS_SYSTEM_DB_PORT=3306 +MYEMS_SYSTEM_DB_DATABASE=myems_system_db +MYEMS_SYSTEM_DB_USER=root +MYEMS_SYSTEM_DB_PASSWORD=!MyEMS1 + +# config for myems_historical_db +MYEMS_HISTORICAL_DB_HOST=127.0.0.1 +MYEMS_HISTORICAL_DB_PORT=3306 +MYEMS_HISTORICAL_DB_DATABASE=myems_historical_db +MYEMS_HISTORICAL_DB_USER=root +MYEMS_HISTORICAL_DB_PASSWORD=!MyEMS1 + +# config for myems_energy_db +MYEMS_ENERGY_DB_HOST=127.0.0.1 +MYEMS_ENERGY_DB_PORT=3306 +MYEMS_ENERGY_DB_DATABASE=myems_energy_db +MYEMS_ENERGY_DB_USER=root +MYEMS_ENERGY_DB_PASSWORD=!MyEMS1 + +# config for myems_billing_db +MYEMS_BILLING_DB_HOST=127.0.0.1 +MYEMS_BILLING_DB_PORT=3306 +MYEMS_BILLING_DB_DATABASE=myems_billing_db +MYEMS_BILLING_DB_USER=root +MYEMS_BILLING_DB_PASSWORD=!MyEMS1 + +# indicates in how many minutes to normalize energy consumption +# 30 for half hourly +# 60 for hourly +MINUTES_TO_COUNT=60 + +# indicates within how many minutes to allow myems-cleaning service to clean the historical data +MINUTES_TO_CLEAN=30 + +# indicates from when (in UTC timezone) to calculate if the energy data is empty or were cleared +# format string: "%Y-%m-%d %H:%M:%S" +START_DATETIME_UTC="2019-12-31 16:00:00" + +# indicates the project's time zone offset from UTC +UTC_OFFSET=+08:00 + +# the number of worker processes in parallel for meter and virtual meter +# the pool size depends on the computing performance of the database server and the analysis server +POOL_SIZE=5 \ No newline at end of file diff --git a/myems-normalization/README.md b/myems-normalization/README.md index 00a3e4f7..ef2f4223 100644 --- a/myems-normalization/README.md +++ b/myems-normalization/README.md @@ -8,11 +8,22 @@ This service is a component of MyEMS and it normalizes energy data in historical ### Prerequisites -mysql.connector +mysql-connector-python + +openpyxl sympy -openpyxl +python-decouple + + +### Quick Run for Development + +```bash +pip install -r requirements.txt +chmod +x run.sh +run.sh +``` ### Installation @@ -22,7 +33,7 @@ cd ~/tools wget https://dev.mysql.com/get/Downloads/Connector-Python/mysql-connector-python-8.0.20.tar.gz tar xzf mysql-connector-python-8.0.20.tar.gz cd ~/tools/mysql-connector-python-8.0.20 -sudo python3 setup.py install +python3 setup.py install ``` Download and install mpmath @@ -30,7 +41,7 @@ Download and install mpmath cd ~/tools git clone https://github.com/fredrik-johansson/mpmath.git cd ~/tools/mpmath -sudo python3 setup.py install +python3 setup.py install ``` Download and install SymPy @@ -38,7 +49,7 @@ Download and install SymPy cd ~/tools git clone https://github.com/sympy/sympy.git cd ~/tools/sympy -sudo python3 setupegg.py develop +python3 setupegg.py develop ``` Download and install openpyxl @@ -62,11 +73,11 @@ tar xzf openpyxl-3.0.7.tar.gz ```bash cd ~/tools/et_xmlfile-1.1 -sudo python3 setup.py install +python3 setup.py install cd ~/tools/jdcal -sudo python3 setup.py install +python3 setup.py install cd ~/tools/openpyxl-3.0.7 -sudo python3 setup.py install +python3 setup.py install ``` Install myems-normalization service: @@ -74,29 +85,29 @@ Install myems-normalization service: cd ~ git clone https://github.com/MyEMS/myems.git cd myems -sudo git checkout master (or the latest release tag) -sudo cp -r ~/myems/myems-normalization /myems-normalization +git checkout master (or the latest release tag) +cp -r ~/myems/myems-normalization /myems-normalization ``` - -Edit config.py -``` -sudo nano /myems-normalization/config.py +Create .env file based on .env.example and edit the .env file if needed: +```bash +cp /myems-normalization/.env.example /myems-cleaning/.env +nano /myems-normalization/.env ``` Setup systemd service: ``` -sudo cp myems-normalization.service /lib/systemd/system/ +cp myems-normalization.service /lib/systemd/system/ ``` Enable the service: ``` -sudo systemctl enable myems-normalization.service +systemctl enable myems-normalization.service ``` Start the service: ``` -sudo systemctl start myems-normalization.service +systemctl start myems-normalization.service ``` Monitor the service: ```bash -sudo systemctl status myems-normalization.service +systemctl status myems-normalization.service ``` View the log: ```bash diff --git a/myems-normalization/config.py b/myems-normalization/config.py index ebfb65ce..b82709fa 100644 --- a/myems-normalization/config.py +++ b/myems-normalization/config.py @@ -1,43 +1,76 @@ +from decouple import config + + +MYEMS_SYSTEM_DB_HOST = config('MYEMS_SYSTEM_DB_HOST', default='127.0.0.1') +MYEMS_SYSTEM_DB_PORT = config('MYEMS_SYSTEM_DB_PORT', default=3306, cast=int) +MYEMS_SYSTEM_DB_DATABASE = config('MYEMS_SYSTEM_DB_DATABASE', default='myems_system_db') +MYEMS_SYSTEM_DB_USER = config('MYEMS_SYSTEM_DB_USER', default='root') +MYEMS_SYSTEM_DB_PASSWORD = config('MYEMS_SYSTEM_DB_PASSWORD', default='!MyEMS1') + +MYEMS_HISTORICAL_DB_HOST = config('MYEMS_HISTORICAL_DB_HOST', default='127.0.0.1') +MYEMS_HISTORICAL_DB_PORT = config('MYEMS_HISTORICAL_DB_PORT', default=3306, cast=int) +MYEMS_HISTORICAL_DB_DATABASE = config('MYEMS_HISTORICAL_DB_DATABASE', default='myems_system_db') +MYEMS_HISTORICAL_DB_USER = config('MYEMS_HISTORICAL_DB_USER', default='root') +MYEMS_HISTORICAL_DB_PASSWORD = config('MYEMS_HISTORICAL_DB_PASSWORD', default='!MyEMS1') + +MYEMS_ENERGY_DB_HOST = config('MYEMS_ENERGY_DB_HOST', default='127.0.0.1') +MYEMS_ENERGY_DB_PORT = config('MYEMS_ENERGY_DB_PORT', default=3306, cast=int) +MYEMS_ENERGY_DB_DATABASE = config('MYEMS_ENERGY_DB_DATABASE', default='myems_system_db') +MYEMS_ENERGY_DB_USER = config('MYEMS_ENERGY_DB_USER', default='root') +MYEMS_ENERGY_DB_PASSWORD = config('MYEMS_ENERGY_DB_PASSWORD', default='!MyEMS1') + +MYEMS_BILLING_DB_HOST = config('MYEMS_BILLING_DB_HOST', default='127.0.0.1') +MYEMS_BILLING_DB_PORT = config('MYEMS_BILLING_DB_PORT', default=3306, cast=int) +MYEMS_BILLING_DB_DATABASE = config('MYEMS_BILLING_DB_DATABASE', default='myems_system_db') +MYEMS_BILLING_DB_USER = config('MYEMS_BILLING_DB_USER', default='root') +MYEMS_BILLING_DB_PASSWORD = config('MYEMS_BILLING_DB_PASSWORD', default='!MyEMS1') + +MINUTES_TO_COUNT = config('MINUTES_TO_COUNT', default=60, cast=int) +MINUTES_TO_CLEAN = config('MINUTES_TO_CLEAN', default=30, cast=int) +START_DATETIME_UTC = config('START_DATETIME_UTC', default='2019-12-31 16:00:00') +UTC_OFFSET = config('UTC_OFFSET', default='+08:00') +POOL_SIZE = config('POOL_SIZE', default=5, cast=int) + myems_system_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_system_db', - 'port': 3306, + 'host': MYEMS_SYSTEM_DB_HOST, + 'port': MYEMS_SYSTEM_DB_PORT, + 'database': MYEMS_SYSTEM_DB_DATABASE, + 'user': MYEMS_SYSTEM_DB_USER, + 'password': MYEMS_SYSTEM_DB_PASSWORD, } myems_energy_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_energy_db', - 'port': 3306, + 'host': MYEMS_ENERGY_DB_HOST, + 'port': MYEMS_ENERGY_DB_PORT, + 'database': MYEMS_ENERGY_DB_DATABASE, + 'user': MYEMS_ENERGY_DB_USER, + 'password': MYEMS_ENERGY_DB_PASSWORD, } myems_historical_db = { - 'user': 'root', - 'password': '!MyEMS1', - 'host': '127.0.0.1', - 'database': 'myems_historical_db', - 'port': 3306, + 'host': MYEMS_HISTORICAL_DB_HOST, + 'port': MYEMS_HISTORICAL_DB_PORT, + 'database': MYEMS_HISTORICAL_DB_DATABASE, + 'user': MYEMS_HISTORICAL_DB_USER, + 'password': MYEMS_HISTORICAL_DB_PASSWORD, } # indicates in how many minutes to normalize energy consumption # 30 for half hourly # 60 for hourly -minutes_to_count = 60 +minutes_to_count = MINUTES_TO_COUNT # indicates within how many minutes to allow myems-cleaning service to clean the historical data -minutes_to_clean = 30 +minutes_to_clean = MINUTES_TO_CLEAN # indicates from when (in UTC timezone) to calculate if the energy data is empty or were cleared -# format string: '%Y-%m-%d %H:%M:%S' -start_datetime_utc = '2019-12-31 16:00:00' +# format string: "%Y-%m-%d %H:%M:%S" +start_datetime_utc = START_DATETIME_UTC + +# indicates the project's time zone offset from UTC +utc_offset = UTC_OFFSET # the number of worker processes in parallel for meter and virtual meter # the pool size depends on the computing performance of the database server and the analysis server -pool_size = 5 - -# indicates the project's time zone offset from UTC -utc_offset = '+08:00' +pool_size = POOL_SIZE diff --git a/myems-normalization/requirements.txt b/myems-normalization/requirements.txt index 67ece48e..f842dd19 100644 --- a/myems-normalization/requirements.txt +++ b/myems-normalization/requirements.txt @@ -1,3 +1,4 @@ -mysql-connector +mysql-connector-python openpyxl -sympy \ No newline at end of file +sympy +python-decouple \ No newline at end of file diff --git a/myems-normalization/run.sh b/myems-normalization/run.sh new file mode 100644 index 00000000..7a84f979 --- /dev/null +++ b/myems-normalization/run.sh @@ -0,0 +1 @@ +python main.py \ No newline at end of file