From cab097fc781472812ee7e06f1a7eb9b42b4af3d4 Mon Sep 17 00:00:00 2001 From: "13621160019@163.com" <13621160019@163.com> Date: Fri, 19 Feb 2021 13:09:42 +0800 Subject: [PATCH] merged myems-aggregation --- myems-aggregation/LICENSE | 21 + myems-aggregation/README.md | 62 + ...mbined_equipment_billing_input_category.py | 270 ++++ .../combined_equipment_billing_input_item.py | 270 ++++ ...bined_equipment_billing_output_category.py | 270 ++++ ...ombined_equipment_energy_input_category.py | 619 ++++++++++ .../combined_equipment_energy_input_item.py | 622 ++++++++++ ...mbined_equipment_energy_output_category.py | 619 ++++++++++ myems-aggregation/config.py | 45 + .../equipment_billing_input_category.py | 269 ++++ .../equipment_billing_input_item.py | 269 ++++ .../equipment_billing_output_category.py | 269 ++++ .../equipment_energy_input_category.py | 524 ++++++++ .../equipment_energy_input_item.py | 527 ++++++++ .../equipment_energy_output_category.py | 524 ++++++++ myems-aggregation/main.py | 147 +++ myems-aggregation/meter_billing.py | 266 ++++ myems-aggregation/myems-aggregation.service | 15 + .../shopfloor_billing_input_category.py | 269 ++++ .../shopfloor_billing_input_item.py | 269 ++++ .../shopfloor_energy_input_category.py | 615 ++++++++++ .../shopfloor_energy_input_item.py | 618 ++++++++++ .../space_billing_input_category.py | 269 ++++ myems-aggregation/space_billing_input_item.py | 269 ++++ .../space_billing_output_category.py | 269 ++++ .../space_energy_input_category.py | 1081 ++++++++++++++++ myems-aggregation/space_energy_input_item.py | 1084 +++++++++++++++++ .../space_energy_output_category.py | 533 ++++++++ .../store_billing_input_category.py | 269 ++++ myems-aggregation/store_billing_input_item.py | 269 ++++ .../store_energy_input_category.py | 521 ++++++++ myems-aggregation/store_energy_input_item.py | 524 ++++++++ myems-aggregation/tariff.py | 205 ++++ .../tenant_billing_input_category.py | 269 ++++ .../tenant_billing_input_item.py | 269 ++++ .../tenant_energy_input_category.py | 521 ++++++++ myems-aggregation/tenant_energy_input_item.py | 524 ++++++++ myems-aggregation/test_tariff.py | 33 + 38 files changed, 14288 insertions(+) create mode 100644 myems-aggregation/LICENSE create mode 100644 myems-aggregation/README.md create mode 100644 myems-aggregation/combined_equipment_billing_input_category.py create mode 100644 myems-aggregation/combined_equipment_billing_input_item.py create mode 100644 myems-aggregation/combined_equipment_billing_output_category.py create mode 100644 myems-aggregation/combined_equipment_energy_input_category.py create mode 100644 myems-aggregation/combined_equipment_energy_input_item.py create mode 100644 myems-aggregation/combined_equipment_energy_output_category.py create mode 100644 myems-aggregation/config.py create mode 100644 myems-aggregation/equipment_billing_input_category.py create mode 100644 myems-aggregation/equipment_billing_input_item.py create mode 100644 myems-aggregation/equipment_billing_output_category.py create mode 100644 myems-aggregation/equipment_energy_input_category.py create mode 100644 myems-aggregation/equipment_energy_input_item.py create mode 100644 myems-aggregation/equipment_energy_output_category.py create mode 100644 myems-aggregation/main.py create mode 100644 myems-aggregation/meter_billing.py create mode 100644 myems-aggregation/myems-aggregation.service create mode 100644 myems-aggregation/shopfloor_billing_input_category.py create mode 100644 myems-aggregation/shopfloor_billing_input_item.py create mode 100644 myems-aggregation/shopfloor_energy_input_category.py create mode 100644 myems-aggregation/shopfloor_energy_input_item.py create mode 100644 myems-aggregation/space_billing_input_category.py create mode 100644 myems-aggregation/space_billing_input_item.py create mode 100644 myems-aggregation/space_billing_output_category.py create mode 100644 myems-aggregation/space_energy_input_category.py create mode 100644 myems-aggregation/space_energy_input_item.py create mode 100644 myems-aggregation/space_energy_output_category.py create mode 100644 myems-aggregation/store_billing_input_category.py create mode 100644 myems-aggregation/store_billing_input_item.py create mode 100644 myems-aggregation/store_energy_input_category.py create mode 100644 myems-aggregation/store_energy_input_item.py create mode 100644 myems-aggregation/tariff.py create mode 100644 myems-aggregation/tenant_billing_input_category.py create mode 100644 myems-aggregation/tenant_billing_input_item.py create mode 100644 myems-aggregation/tenant_energy_input_category.py create mode 100644 myems-aggregation/tenant_energy_input_item.py create mode 100644 myems-aggregation/test_tariff.py diff --git a/myems-aggregation/LICENSE b/myems-aggregation/LICENSE new file mode 100644 index 00000000..b91c1ac4 --- /dev/null +++ b/myems-aggregation/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 MyEMS + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/myems-aggregation/README.md b/myems-aggregation/README.md new file mode 100644 index 00000000..eb680e17 --- /dev/null +++ b/myems-aggregation/README.md @@ -0,0 +1,62 @@ +## MyEMS Aggregation Service 数据汇总服务 + + + +### Introduction + +This service is a component of MyEMS and it aggregates normalized data up to multiple dimensions. + +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/cb75cee835ba46118115e088f8be6d87)](https://app.codacy.com/gh/myems/myems-aggregation?utm_source=github.com&utm_medium=referral&utm_content=myems/myems-aggregation&utm_campaign=Badge_Grade) +[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/myems/myems-aggregation/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/myems/myems-aggregation/?branch=master) +[![Maintainability](https://api.codeclimate.com/v1/badges/ecff11174fd74975946c/maintainability)](https://codeclimate.com/github/myems/myems-aggregation/maintainability) +[![Total alerts](https://img.shields.io/lgtm/alerts/g/myems/myems-bacnet.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/myems/myems-bacnet/alerts/) + +### Prerequisites + +mysql.connector + + + +### Installation + +Download and install MySQL Connector: +``` + $ cd ~/tools + $ wget https://dev.mysql.com/get/Downloads/Connector-Python/mysql-connector-python-8.0.20.tar.gz + $ tar xzf mysql-connector-python-8.0.20.tar.gz + $ cd ~/tools/mysql-connector-python-8.0.20 + $ sudo python3 setup.py install +``` + +Install myems-aggregation service: +``` + $ cd ~ + $ git clone https://github.com/myems/myems.git + $ cd myems + $ sudo git checkout master (or the latest release tag) + $ sudo cp -R ~/myems/myems-aggregation /myems-aggregation +``` + Edit config.py for your project +``` + $ sudo nano /myems-aggregation/config.py +``` + + Setup systemd service: +``` + $ sudo cp myems-aggregation.service /lib/systemd/system/ +``` + Enable the service: +``` + $ sudo systemctl enable myems-aggregation.service +``` + + Start the service: +``` + $ sudo systemctl start myems-aggregation.service +``` + +### References + +[1]. https://myems.io + +[2]. https://dev.mysql.com/doc/connector-python/en/ diff --git a/myems-aggregation/combined_equipment_billing_input_category.py b/myems-aggregation/combined_equipment_billing_input_category.py new file mode 100644 index 00000000..581d32e6 --- /dev/null +++ b/myems-aggregation/combined_equipment_billing_input_category.py @@ -0,0 +1,270 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +import tariff +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all combined equipments +# for each combined equipment in list: +# Step 2: get the latest start_datetime_utc +# Step 3: get all energy input data since the latest start_datetime_utc +# Step 4: get tariffs +# Step 5: calculate billing by multiplying energy with tariff +# Step 6: save billing data to database +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all combined equipments + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of combined_equipment_billing_input_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name, cost_center_id " + " FROM tbl_combined_equipments " + " ORDER BY id ") + rows_combined_equipments = cursor_system_db.fetchall() + + if rows_combined_equipments is None or len(rows_combined_equipments) == 0: + print("Step 1.2: There isn't any combined equipments. ") + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + combined_equipment_list = list() + for row in rows_combined_equipments: + combined_equipment_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]}) + + except Exception as e: + logger.error("Error in step 1.2 of combined_equipment_billing_input_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Step 1.2: Got all combined_equipments from MyEMS System Database") + + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + logger.error("Error in step 1.3 of combined_equipment_billing_input_category " + str(e)) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Energy Database") + + cnx_billing_db = None + cursor_billing_db = None + try: + cnx_billing_db = mysql.connector.connect(**config.myems_billing_db) + cursor_billing_db = cnx_billing_db.cursor() + except Exception as e: + logger.error("Error in step 1.4 of combined_equipment_billing_input_category " + str(e)) + if cursor_billing_db: + cursor_billing_db.close() + if cnx_billing_db: + cnx_billing_db.close() + + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Billing Database") + + for combined_equipment in combined_equipment_list: + + ############################################################################################################ + # Step 2: get the latest start_datetime_utc + ############################################################################################################ + print("Step 2: get the latest start_datetime_utc from billing database for " + combined_equipment['name']) + try: + cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) " + " FROM tbl_combined_equipment_input_category_hourly " + " WHERE combined_equipment_id = %s ", + (combined_equipment['id'], )) + row_datetime = cursor_billing_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]) + except Exception as e: + logger.error("Error in step 2 of combined_equipment_billing_input_category " + str(e)) + # break the for combined equipment loop + break + + ############################################################################################################ + # Step 3: get all energy input data since the latest start_datetime_utc + ############################################################################################################ + print("Step 3: get all energy input data since the latest start_datetime_utc") + + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_combined_equipment_input_category_hourly " + " WHERE combined_equipment_id = %s AND start_datetime_utc >= %s " + " ORDER BY id ") + cursor_energy_db.execute(query, (combined_equipment['id'], start_datetime_utc, )) + rows_hourly = cursor_energy_db.fetchall() + + if rows_hourly is None or len(rows_hourly) == 0: + print("Step 3: There isn't any energy input data to calculate. ") + # continue the for combined equipment loop + continue + + energy_dict = dict() + energy_category_list = list() + end_datetime_utc = start_datetime_utc + for row_hourly in rows_hourly: + current_datetime_utc = row_hourly[0] + energy_category_id = row_hourly[1] + + if energy_category_id not in energy_category_list: + energy_category_list.append(energy_category_id) + + actual_value = row_hourly[2] + if energy_dict.get(current_datetime_utc) is None: + energy_dict[current_datetime_utc] = dict() + energy_dict[current_datetime_utc][energy_category_id] = actual_value + if current_datetime_utc > end_datetime_utc: + end_datetime_utc = current_datetime_utc + + ############################################################################################################ + # Step 4: get tariffs + ############################################################################################################ + print("Step 4: get tariffs") + tariff_dict = dict() + for energy_category_id in energy_category_list: + tariff_dict[energy_category_id] = \ + tariff.get_energy_category_tariffs(combined_equipment['cost_center_id'], + energy_category_id, + start_datetime_utc, + end_datetime_utc) + ############################################################################################################ + # Step 5: calculate billing by multiplying energy with tariff + ############################################################################################################ + print("Step 5: calculate billing by multiplying energy with tariff") + billing_dict = dict() + + if len(energy_dict) > 0: + for current_datetime_utc in energy_dict.keys(): + billing_dict[current_datetime_utc] = dict() + for energy_category_id in energy_category_list: + current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc) + current_energy = energy_dict[current_datetime_utc].get(energy_category_id) + if current_tariff is not None \ + and isinstance(current_tariff, Decimal) \ + and current_energy is not None \ + and isinstance(current_energy, Decimal): + billing_dict[current_datetime_utc][energy_category_id] = \ + current_energy * current_tariff + + if len(billing_dict[current_datetime_utc]) == 0: + del billing_dict[current_datetime_utc] + + ############################################################################################################ + # Step 6: save billing data to billing database + ############################################################################################################ + print("Step 6: save billing data to billing database") + + if len(billing_dict) > 0: + try: + add_values = (" INSERT INTO tbl_combined_equipment_input_category_hourly " + " (combined_equipment_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for current_datetime_utc in billing_dict: + for energy_category_id in energy_category_list: + current_billing = billing_dict[current_datetime_utc].get(energy_category_id) + if current_billing is not None and isinstance(current_billing, Decimal): + add_values += " (" + str(combined_equipment['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + current_datetime_utc.isoformat()[0:19] + "'," + add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_billing_db.execute(add_values[:-2]) + cnx_billing_db.commit() + except Exception as e: + logger.error("Error in step 6 of combined_equipment_billing_input_category " + str(e)) + # break the for combined equipment loop + break + + # end of for combined equipment loop + if cnx_system_db: + cnx_system_db.close() + if cursor_system_db: + cursor_system_db.close() + + if cnx_energy_db: + cnx_energy_db.close() + if cursor_energy_db: + cursor_energy_db.close() + + if cnx_billing_db: + cnx_billing_db.close() + if cursor_billing_db: + cursor_billing_db.close() + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of the outermost while loop diff --git a/myems-aggregation/combined_equipment_billing_input_item.py b/myems-aggregation/combined_equipment_billing_input_item.py new file mode 100644 index 00000000..d7d14f18 --- /dev/null +++ b/myems-aggregation/combined_equipment_billing_input_item.py @@ -0,0 +1,270 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +import tariff +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all combined equipments +# for each combined equipment in list: +# Step 2: get the latest start_datetime_utc +# Step 3: get all energy input data since the latest start_datetime_utc +# Step 4: get tariffs +# Step 5: calculate billing by multiplying energy with tariff +# Step 6: save billing data to database +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all combined equipments + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of combined_equipment_billing_input_item " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name, cost_center_id " + " FROM tbl_combined_equipments " + " ORDER BY id ") + rows_combined_equipments = cursor_system_db.fetchall() + + if rows_combined_equipments is None or len(rows_combined_equipments) == 0: + print("Step 1.2: There isn't any combined equipments. ") + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + combined_equipment_list = list() + for row in rows_combined_equipments: + combined_equipment_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]}) + + except Exception as e: + logger.error("Error in step 1.2 of combined_equipment_billing_input_item " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Step 1.2: Got all combined equipments from MyEMS System Database") + + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + logger.error("Error in step 1.3 of combined_equipment_billing_input_item " + str(e)) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Energy Database") + + cnx_billing_db = None + cursor_billing_db = None + try: + cnx_billing_db = mysql.connector.connect(**config.myems_billing_db) + cursor_billing_db = cnx_billing_db.cursor() + except Exception as e: + logger.error("Error in step 1.4 of combined_equipment_billing_input_item " + str(e)) + if cursor_billing_db: + cursor_billing_db.close() + if cnx_billing_db: + cnx_billing_db.close() + + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Billing Database") + + for combined_equipment in combined_equipment_list: + + ############################################################################################################ + # Step 2: get the latest start_datetime_utc + ############################################################################################################ + print("Step 2: get the latest start_datetime_utc from billing database for " + combined_equipment['name']) + try: + cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) " + " FROM tbl_combined_equipment_input_item_hourly " + " WHERE combined_equipment_id = %s ", + (combined_equipment['id'], )) + row_datetime = cursor_billing_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]) + except Exception as e: + logger.error("Error in step 2 of combined_equipment_billing_input_item " + str(e)) + # break the for combined equipment loop + break + + ############################################################################################################ + # Step 3: get all energy input data since the latest start_datetime_utc + ############################################################################################################ + print("Step 3: get all energy input data since the latest start_datetime_utc") + + query = (" SELECT start_datetime_utc, energy_item_id, actual_value " + " FROM tbl_combined_equipment_input_item_hourly " + " WHERE combined_equipment_id = %s AND start_datetime_utc >= %s " + " ORDER BY id ") + cursor_energy_db.execute(query, (combined_equipment['id'], start_datetime_utc, )) + rows_hourly = cursor_energy_db.fetchall() + + if rows_hourly is None or len(rows_hourly) == 0: + print("Step 3: There isn't any energy input data to calculate. ") + # continue the for combined equipment loop + continue + + energy_dict = dict() + energy_item_list = list() + end_datetime_utc = start_datetime_utc + for row_hourly in rows_hourly: + current_datetime_utc = row_hourly[0] + energy_item_id = row_hourly[1] + + if energy_item_id not in energy_item_list: + energy_item_list.append(energy_item_id) + + actual_value = row_hourly[2] + if energy_dict.get(current_datetime_utc) is None: + energy_dict[current_datetime_utc] = dict() + energy_dict[current_datetime_utc][energy_item_id] = actual_value + if current_datetime_utc > end_datetime_utc: + end_datetime_utc = current_datetime_utc + + ############################################################################################################ + # Step 4: get tariffs + ############################################################################################################ + print("Step 4: get tariffs") + tariff_dict = dict() + for energy_item_id in energy_item_list: + tariff_dict[energy_item_id] = \ + tariff.get_energy_item_tariffs(combined_equipment['cost_center_id'], + energy_item_id, + start_datetime_utc, + end_datetime_utc) + ############################################################################################################ + # Step 5: calculate billing by multiplying energy with tariff + ############################################################################################################ + print("Step 5: calculate billing by multiplying energy with tariff") + billing_dict = dict() + + if len(energy_dict) > 0: + for current_datetime_utc in energy_dict.keys(): + billing_dict[current_datetime_utc] = dict() + for energy_item_id in energy_item_list: + current_tariff = tariff_dict[energy_item_id].get(current_datetime_utc) + current_energy = energy_dict[current_datetime_utc].get(energy_item_id) + if current_tariff is not None \ + and isinstance(current_tariff, Decimal) \ + and current_energy is not None \ + and isinstance(current_energy, Decimal): + billing_dict[current_datetime_utc][energy_item_id] = \ + current_energy * current_tariff + + if len(billing_dict[current_datetime_utc]) == 0: + del billing_dict[current_datetime_utc] + + ############################################################################################################ + # Step 6: save billing data to billing database + ############################################################################################################ + print("Step 6: save billing data to billing database") + + if len(billing_dict) > 0: + try: + add_values = (" INSERT INTO tbl_combined_equipment_input_item_hourly " + " (combined_equipment_id, " + " energy_item_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for current_datetime_utc in billing_dict: + for energy_item_id in energy_item_list: + current_billing = billing_dict[current_datetime_utc].get(energy_item_id) + if current_billing is not None and isinstance(current_billing, Decimal): + add_values += " (" + str(combined_equipment['id']) + "," + add_values += " " + str(energy_item_id) + "," + add_values += "'" + current_datetime_utc.isoformat()[0:19] + "'," + add_values += str(billing_dict[current_datetime_utc][energy_item_id]) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_billing_db.execute(add_values[:-2]) + cnx_billing_db.commit() + except Exception as e: + logger.error("Error in step 6 of combined_equipment_billing_input_item " + str(e)) + # break the for combined equipment loop + break + + # end of for combined equipment loop + if cnx_system_db: + cnx_system_db.close() + if cursor_system_db: + cursor_system_db.close() + + if cnx_energy_db: + cnx_energy_db.close() + if cursor_energy_db: + cursor_energy_db.close() + + if cnx_billing_db: + cnx_billing_db.close() + if cursor_billing_db: + cursor_billing_db.close() + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of the outermost while loop diff --git a/myems-aggregation/combined_equipment_billing_output_category.py b/myems-aggregation/combined_equipment_billing_output_category.py new file mode 100644 index 00000000..c8d520a6 --- /dev/null +++ b/myems-aggregation/combined_equipment_billing_output_category.py @@ -0,0 +1,270 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +import tariff +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all combined equipments +# for each combined equipment in list: +# Step 2: get the latest start_datetime_utc +# Step 3: get all energy input data since the latest start_datetime_utc +# Step 4: get tariffs +# Step 5: calculate billing by multiplying energy with tariff +# Step 6: save billing data to database +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all combined equipments + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of combined_equipment_billing_input_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name, cost_center_id " + " FROM tbl_combined_equipments " + " ORDER BY id ") + rows_combined_equipments = cursor_system_db.fetchall() + + if rows_combined_equipments is None or len(rows_combined_equipments) == 0: + print("Step 1.2: There isn't any combined equipments. ") + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + combined_equipment_list = list() + for row in rows_combined_equipments: + combined_equipment_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]}) + + except Exception as e: + logger.error("Error in step 1.2 of combined_equipment_billing_output_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Step 1.2: Got all combined equipments from MyEMS System Database") + + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + logger.error("Error in step 1.3 of combined_equipment_billing_output_category " + str(e)) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Energy Database") + + cnx_billing_db = None + cursor_billing_db = None + try: + cnx_billing_db = mysql.connector.connect(**config.myems_billing_db) + cursor_billing_db = cnx_billing_db.cursor() + except Exception as e: + logger.error("Error in step 1.4 of combined_equipment_billing_output_category " + str(e)) + if cursor_billing_db: + cursor_billing_db.close() + if cnx_billing_db: + cnx_billing_db.close() + + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Billing Database") + + for combined_equipment in combined_equipment_list: + + ############################################################################################################ + # Step 2: get the latest start_datetime_utc + ############################################################################################################ + print("Step 2: get the latest start_datetime_utc from billing database for " + combined_equipment['name']) + try: + cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) " + " FROM tbl_combined_equipment_output_category_hourly " + " WHERE combined_equipment_id = %s ", + (combined_equipment['id'], )) + row_datetime = cursor_billing_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]) + except Exception as e: + logger.error("Error in step 2 of combined_equipment_billing_output_category " + str(e)) + # break the for combined equipment loop + break + + ############################################################################################################ + # Step 3: get all energy output data since the latest start_datetime_utc + ############################################################################################################ + print("Step 3: get all energy output data since the latest start_datetime_utc") + + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_combined_equipment_output_category_hourly " + " WHERE combined_equipment_id = %s AND start_datetime_utc >= %s " + " ORDER BY id ") + cursor_energy_db.execute(query, (combined_equipment['id'], start_datetime_utc, )) + rows_hourly = cursor_energy_db.fetchall() + + if rows_hourly is None or len(rows_hourly) == 0: + print("Step 3: There isn't any energy output data to calculate. ") + # continue the for combined equipment loop + continue + + energy_dict = dict() + energy_category_list = list() + end_datetime_utc = start_datetime_utc + for row_hourly in rows_hourly: + current_datetime_utc = row_hourly[0] + energy_category_id = row_hourly[1] + + if energy_category_id not in energy_category_list: + energy_category_list.append(energy_category_id) + + actual_value = row_hourly[2] + if energy_dict.get(current_datetime_utc) is None: + energy_dict[current_datetime_utc] = dict() + energy_dict[current_datetime_utc][energy_category_id] = actual_value + if current_datetime_utc > end_datetime_utc: + end_datetime_utc = current_datetime_utc + + ############################################################################################################ + # Step 4: get tariffs + ############################################################################################################ + print("Step 4: get tariffs") + tariff_dict = dict() + for energy_category_id in energy_category_list: + tariff_dict[energy_category_id] = \ + tariff.get_energy_category_tariffs(combined_equipment['cost_center_id'], + energy_category_id, + start_datetime_utc, + end_datetime_utc) + ############################################################################################################ + # Step 5: calculate billing by multiplying energy with tariff + ############################################################################################################ + print("Step 5: calculate billing by multiplying energy with tariff") + billing_dict = dict() + + if len(energy_dict) > 0: + for current_datetime_utc in energy_dict.keys(): + billing_dict[current_datetime_utc] = dict() + for energy_category_id in energy_category_list: + current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc) + current_energy = energy_dict[current_datetime_utc].get(energy_category_id) + if current_tariff is not None \ + and isinstance(current_tariff, Decimal) \ + and current_energy is not None \ + and isinstance(current_energy, Decimal): + billing_dict[current_datetime_utc][energy_category_id] = \ + current_energy * current_tariff + + if len(billing_dict[current_datetime_utc]) == 0: + del billing_dict[current_datetime_utc] + + ############################################################################################################ + # Step 6: save billing data to billing database + ############################################################################################################ + print("Step 6: save billing data to billing database") + + if len(billing_dict) > 0: + try: + add_values = (" INSERT INTO tbl_combined_equipment_output_category_hourly " + " (combined_equipment_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for current_datetime_utc in billing_dict: + for energy_category_id in energy_category_list: + current_billing = billing_dict[current_datetime_utc].get(energy_category_id) + if current_billing is not None and isinstance(current_billing, Decimal): + add_values += " (" + str(combined_equipment['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + current_datetime_utc.isoformat()[0:19] + "'," + add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_billing_db.execute(add_values[:-2]) + cnx_billing_db.commit() + except Exception as e: + logger.error("Error in step 6 of combined_equipment_billing_output_category " + str(e)) + # break the for combined_equipment loop + break + + # end of for combined equipment loop + if cnx_system_db: + cnx_system_db.close() + if cursor_system_db: + cursor_system_db.close() + + if cnx_energy_db: + cnx_energy_db.close() + if cursor_energy_db: + cursor_energy_db.close() + + if cnx_billing_db: + cnx_billing_db.close() + if cursor_billing_db: + cursor_billing_db.close() + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of the outermost while loop diff --git a/myems-aggregation/combined_equipment_energy_input_category.py b/myems-aggregation/combined_equipment_energy_input_category.py new file mode 100644 index 00000000..b03a93eb --- /dev/null +++ b/myems-aggregation/combined_equipment_energy_input_category.py @@ -0,0 +1,619 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +from multiprocessing import Pool +import random +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all combined equipments +# Step 2: Create multiprocessing pool to call worker in parallel +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all combined equipments + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of combined_equipment_energy_input_category.main " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_combined_equipments " + " ORDER BY id ") + rows_combined_equipments = cursor_system_db.fetchall() + + if rows_combined_equipments is None or len(rows_combined_equipments) == 0: + print("There isn't any combined equipments ") + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + + combined_equipment_list = list() + for row in rows_combined_equipments: + combined_equipment_list.append({"id": row[0], "name": row[1]}) + + except Exception as e: + logger.error("Error in step 1.2 of combined_equipment_energy_input_category.main " + str(e)) + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + print("Got all combined equipments in MyEMS System Database") + + # shuffle the combined equipment list for randomly calculating the meter hourly value + random.shuffle(combined_equipment_list) + + ################################################################################################################ + # Step 2: Create multiprocessing pool to call worker in parallel + ################################################################################################################ + p = Pool(processes=config.pool_size) + error_list = p.map(worker, combined_equipment_list) + p.close() + p.join() + + for error in error_list: + if error is not None and len(error) > 0: + logger.error(error) + + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of outer while + + +######################################################################################################################## +# PROCEDURES: +# Step 1: get all input meters associated with the combined equipment +# Step 2: get all input virtual meters associated with the combined equipment +# Step 3: get all input offline meters associated with the combined equipment +# Step 4: get all equipments associated with the combined equipment +# Step 5: determine start datetime and end datetime to aggregate +# Step 6: for each meter in list, get energy input data from energy database +# Step 7: for each virtual meter in list, get energy input data from energy database +# Step 8: for each offline meter in list, get energy input data from energy database +# Step 9: for each equipment in list, get energy input data from energy database +# Step 10: determine common time slot to aggregate +# Step 11: aggregate energy data in the common time slot by energy categories and hourly +# Step 12: save energy data to energy database +# +# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter +######################################################################################################################## + +def worker(combined_equipment): + #################################################################################################################### + # Step 1: get all input meters associated with the combined equipment + #################################################################################################################### + print("Step 1: get all input meters associated with the combined equipment " + str(combined_equipment['name'])) + + meter_list = list() + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + error_string = "Error in step 1.1 of combined_equipment_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_meters m, tbl_combined_equipments_meters em " + " WHERE m.id = em.meter_id " + " AND m.is_counted = true " + " AND em.is_output = false " + " AND em.combined_equipment_id = %s ", + (combined_equipment['id'],)) + rows_meters = cursor_system_db.fetchall() + + if rows_meters is not None and len(rows_meters) > 0: + for row in rows_meters: + meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 1.2 of combined_equipment_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 2: get all input virtual meters associated with the combined equipment + #################################################################################################################### + print("Step 2: get all input virtual meters associated with the combined equipment") + virtual_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_virtual_meters m, tbl_combined_equipments_virtual_meters em " + " WHERE m.id = em.virtual_meter_id " + " AND m.is_counted = true " + " AND em.is_output = false " + " AND em.combined_equipment_id = %s ", + (combined_equipment['id'],)) + rows_virtual_meters = cursor_system_db.fetchall() + + if rows_virtual_meters is not None and len(rows_virtual_meters) > 0: + for row in rows_virtual_meters: + virtual_meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 2.1 of combined_equipment_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 3: get all input offline meters associated with the combined equipment + #################################################################################################################### + print("Step 3: get all input offline meters associated with the combined equipment") + + offline_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_offline_meters m, tbl_combined_equipments_offline_meters em " + " WHERE m.id = em.offline_meter_id " + " AND m.is_counted = true " + " AND em.is_output = false " + " AND em.combined_equipment_id = %s ", + (combined_equipment['id'],)) + rows_offline_meters = cursor_system_db.fetchall() + + if rows_offline_meters is not None and len(rows_offline_meters) > 0: + for row in rows_offline_meters: + offline_meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 3.1 of combined_equipment_energy_input_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + #################################################################################################################### + # Step 4: get all equipments associated with the combined equipment + #################################################################################################################### + print("Step 4: get all equipments associated with the combined equipment") + + equipment_list = list() + + try: + cursor_system_db.execute(" SELECT e.id, e.name " + " FROM tbl_equipments e, tbl_combined_equipments_equipments ce " + " WHERE e.id = ce.equipment_id " + " AND e.is_input_counted = true " + " AND ce.combined_equipment_id = %s ", + (combined_equipment['id'],)) + rows_equipments = cursor_system_db.fetchall() + + if rows_equipments is not None and len(rows_equipments) > 0: + for row in rows_equipments: + equipment_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 4 of combined_equipment_energy_input_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + #################################################################################################################### + # stop to the next combined equipment if this combined equipment is empty + #################################################################################################################### + if (meter_list is None or len(meter_list) == 0) and \ + (virtual_meter_list is None or len(virtual_meter_list) == 0) and \ + (offline_meter_list is None or len(offline_meter_list) == 0) and \ + (equipment_list is None or len(equipment_list) == 0): + print("This is an empty combined equipment ") + return None + + #################################################################################################################### + # Step 5: determine start datetime and end datetime to aggregate + #################################################################################################################### + print("Step 5: determine start datetime and end datetime to aggregate") + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + error_string = "Error in step 5.1 of combined_equipment_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + try: + query = (" SELECT MAX(start_datetime_utc) " + " FROM tbl_combined_equipment_input_category_hourly " + " WHERE combined_equipment_id = %s ") + cursor_energy_db.execute(query, (combined_equipment['id'],)) + row_datetime = cursor_energy_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19] + + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19]) + + except Exception as e: + error_string = "Error in step 5.2 of combined_equipment_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 6: for each meter in list, get energy input data from energy database + #################################################################################################################### + energy_meter_hourly = dict() + try: + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_meter_hourly " + " WHERE meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_meter_hourly[meter_id] = None + else: + energy_meter_hourly[meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 6.1 of combined_equipment_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 7: for each virtual meter in list, get energy input data from energy database + #################################################################################################################### + energy_virtual_meter_hourly = dict() + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + try: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_virtual_meter_hourly " + " WHERE virtual_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_virtual_meter_hourly[virtual_meter_id] = None + else: + energy_virtual_meter_hourly[virtual_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 7.1 of combined_equipment_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 8: for each offline meter in list, get energy input data from energy database + #################################################################################################################### + energy_offline_meter_hourly = dict() + if offline_meter_list is not None and len(offline_meter_list) > 0: + try: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_offline_meter_hourly " + " WHERE offline_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_offline_meter_hourly[offline_meter_id] = None + else: + energy_offline_meter_hourly[offline_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1] + + except Exception as e: + error_string = "Error in step 8.1 of combined_equipment_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 9: for each equipment in list, get energy input data from energy database + #################################################################################################################### + energy_equipment_hourly = dict() + if equipment_list is not None and len(equipment_list) > 0: + try: + for equipment in equipment_list: + equipment_id = str(equipment['id']) + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_equipment_input_category_hourly " + " WHERE equipment_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (equipment_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_equipment_hourly[equipment_id] = None + else: + energy_equipment_hourly[equipment_id] = dict() + for row_value in rows_energy_values: + current_datetime_utc = row_value[0] + if current_datetime_utc not in energy_equipment_hourly[equipment_id]: + energy_equipment_hourly[equipment_id][current_datetime_utc] = dict() + energy_category_id = row_value[1] + actual_value = row_value[2] + energy_equipment_hourly[equipment_id][current_datetime_utc][energy_category_id] = \ + actual_value + except Exception as e: + error_string = "Error in step 9 of combined_equipment_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 10: determine common time slot to aggregate + #################################################################################################################### + + common_start_datetime_utc = start_datetime_utc + common_end_datetime_utc = end_datetime_utc + + print("Getting common time slot of energy values for all meters") + if energy_meter_hourly is not None and len(energy_meter_hourly) > 0: + for meter_id, energy_hourly in energy_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all virtual meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0: + for meter_id, energy_hourly in energy_virtual_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all offline meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0: + for meter_id, energy_hourly in energy_offline_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all equipments...") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_equipment_hourly is not None and len(energy_equipment_hourly) > 0: + for equipment_id, energy_hourly in energy_equipment_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \ + (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \ + (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0) and \ + (energy_equipment_hourly is None or len(energy_equipment_hourly) == 0): + # There isn't any energy data + print("There isn't any energy data") + # continue the for combined equipment loop to the next combined equipment + print("continue the for combined equipment loop to the next combined equipment") + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + return None + + print("common_start_datetime_utc: " + str(common_start_datetime_utc)) + print("common_end_datetime_utc: " + str(common_end_datetime_utc)) + + #################################################################################################################### + # Step 11: aggregate energy data in the common time slot by energy categories and hourly + #################################################################################################################### + + print("Step 11: aggregate energy data in the common time slot by energy categories and hourly") + aggregated_values = list() + try: + current_datetime_utc = common_start_datetime_utc + while common_start_datetime_utc is not None \ + and common_end_datetime_utc is not None \ + and current_datetime_utc <= common_end_datetime_utc: + aggregated_value = dict() + aggregated_value['start_datetime_utc'] = current_datetime_utc + aggregated_value['meta_data'] = dict() + + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + energy_category_id = meter['energy_category_id'] + actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + energy_category_id = virtual_meter['energy_category_id'] + actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if offline_meter_list is not None and len(offline_meter_list) > 0: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + energy_category_id = offline_meter['energy_category_id'] + actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if equipment_list is not None and len(equipment_list) > 0: + for equipment in equipment_list: + equipment_id = str(equipment['id']) + meta_data_dict = energy_equipment_hourly[equipment_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_category_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + aggregated_values.append(aggregated_value) + + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + except Exception as e: + error_string = "Error in step 11 of combined_equipment_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 12: save energy data to energy database + #################################################################################################################### + print("Step 12: save energy data to energy database") + + if len(aggregated_values) > 0: + try: + add_values = (" INSERT INTO tbl_combined_equipment_input_category_hourly " + " (combined_equipment_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for aggregated_value in aggregated_values: + for energy_category_id, actual_value in aggregated_value['meta_data'].items(): + add_values += " (" + str(combined_equipment['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "'," + add_values += str(actual_value) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_energy_db.execute(add_values[:-2]) + cnx_energy_db.commit() + + except Exception as e: + error_string = "Error in step 12.1 of combined_equipment_energy_input_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + else: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() diff --git a/myems-aggregation/combined_equipment_energy_input_item.py b/myems-aggregation/combined_equipment_energy_input_item.py new file mode 100644 index 00000000..570b4bf5 --- /dev/null +++ b/myems-aggregation/combined_equipment_energy_input_item.py @@ -0,0 +1,622 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +from multiprocessing import Pool +import random +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all combined equipments +# Step 2: Create multiprocessing pool to call worker in parallel +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all combined equipments + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of combined_equipment_energy_input_item.main " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_combined_equipments " + " ORDER BY id ") + rows_combined_equipments = cursor_system_db.fetchall() + + if rows_combined_equipments is None or len(rows_combined_equipments) == 0: + print("There isn't any combined equipments ") + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + + combined_equipment_list = list() + for row in rows_combined_equipments: + combined_equipment_list.append({"id": row[0], "name": row[1]}) + + except Exception as e: + logger.error("Error in step 1.2 of combined_equipment_energy_input_item.main " + str(e)) + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + print("Got all combined equipments in MyEMS System Database") + + # shuffle the combined equipment list for randomly calculating the meter hourly value + random.shuffle(combined_equipment_list) + + ################################################################################################################ + # Step 2: Create multiprocessing pool to call worker in parallel + ################################################################################################################ + p = Pool(processes=config.pool_size) + error_list = p.map(worker, combined_equipment_list) + p.close() + p.join() + + for error in error_list: + if error is not None and len(error) > 0: + logger.error(error) + + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of outer while + + +######################################################################################################################## +# PROCEDURES: +# Step 1: get all input meters associated with the combined equipment +# Step 2: get all input virtual meters associated with the combined equipment +# Step 3: get all input offline meters associated with the combined equipment +# Step 4: get all equipments associated with the combined equipment +# Step 5: determine start datetime and end datetime to aggregate +# Step 6: for each meter in list, get energy input data from energy database +# Step 7: for each virtual meter in list, get energy input data from energy database +# Step 8: for each offline meter in list, get energy input data from energy database +# Step 9: for each equipment in list, get energy input data from energy database +# Step 10: determine common time slot to aggregate +# Step 11: aggregate energy data in the common time slot by energy items and hourly +# Step 12: save energy data to energy database +# +# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter +######################################################################################################################## + +def worker(combined_equipment): + #################################################################################################################### + # Step 1: get all input meters associated with the combined equipment + #################################################################################################################### + print("Step 1: get all input meters associated with the combined equipment " + str(combined_equipment['name'])) + + meter_list = list() + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + error_string = "Error in step 1.1 of combined_equipment_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_meters m, tbl_combined_equipments_meters em " + " WHERE m.id = em.meter_id " + " AND m.is_counted = true " + " AND m.energy_item_id is NOT NULL " + " AND em.is_output = false " + " AND em.combined_equipment_id = %s ", + (combined_equipment['id'],)) + rows_meters = cursor_system_db.fetchall() + + if rows_meters is not None and len(rows_meters) > 0: + for row in rows_meters: + meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 1.2 of combined_equipment_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 2: get all input virtual meters associated with the combined equipment + #################################################################################################################### + print("Step 2: get all input virtual meters associated with the combined equipment") + virtual_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_virtual_meters m, tbl_combined_equipments_virtual_meters em " + " WHERE m.id = em.virtual_meter_id " + " AND m.energy_item_id is NOT NULL " + " AND m.is_counted = true " + " AND em.is_output = false " + " AND em.combined_equipment_id = %s ", + (combined_equipment['id'],)) + rows_virtual_meters = cursor_system_db.fetchall() + + if rows_virtual_meters is not None and len(rows_virtual_meters) > 0: + for row in rows_virtual_meters: + virtual_meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 2.1 of combined_equipment_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 3: get all input offline meters associated with the combined equipment + #################################################################################################################### + print("Step 3: get all input offline meters associated with the combined equipment") + + offline_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_offline_meters m, tbl_combined_equipments_offline_meters em " + " WHERE m.id = em.offline_meter_id " + " AND m.energy_item_id is NOT NULL " + " AND m.is_counted = true " + " AND em.is_output = false " + " AND em.combined_equipment_id = %s ", + (combined_equipment['id'],)) + rows_offline_meters = cursor_system_db.fetchall() + + if rows_offline_meters is not None and len(rows_offline_meters) > 0: + for row in rows_offline_meters: + offline_meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 3.1 of combined_equipment_energy_input_item.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + #################################################################################################################### + # Step 4: get all equipments associated with the combined equipment + #################################################################################################################### + print("Step 4: get all equipments associated with the combined equipment") + + equipment_list = list() + + try: + cursor_system_db.execute(" SELECT e.id, e.name " + " FROM tbl_equipments e, tbl_combined_equipments_equipments ce " + " WHERE e.id = ce.equipment_id " + " AND e.is_input_counted = true " + " AND ce.combined_equipment_id = %s ", + (combined_equipment['id'],)) + rows_equipments = cursor_system_db.fetchall() + + if rows_equipments is not None and len(rows_equipments) > 0: + for row in rows_equipments: + equipment_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 4 of combined_equipment_energy_input_item.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + #################################################################################################################### + # stop to the next combined equipment if this combined equipment is empty + #################################################################################################################### + if (meter_list is None or len(meter_list) == 0) and \ + (virtual_meter_list is None or len(virtual_meter_list) == 0) and \ + (offline_meter_list is None or len(offline_meter_list) == 0) and \ + (equipment_list is None or len(equipment_list) == 0): + print("This is an empty combined equipment ") + return None + + #################################################################################################################### + # Step 5: determine start datetime and end datetime to aggregate + #################################################################################################################### + print("Step 5: determine start datetime and end datetime to aggregate") + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + error_string = "Error in step 5.1 of combined_equipment_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + try: + query = (" SELECT MAX(start_datetime_utc) " + " FROM tbl_combined_equipment_input_item_hourly " + " WHERE combined_equipment_id = %s ") + cursor_energy_db.execute(query, (combined_equipment['id'],)) + row_datetime = cursor_energy_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19] + + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19]) + + except Exception as e: + error_string = "Error in step 5.2 of combined_equipment_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 6: for each meter in list, get energy input data from energy database + #################################################################################################################### + energy_meter_hourly = dict() + try: + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_meter_hourly " + " WHERE meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_meter_hourly[meter_id] = None + else: + energy_meter_hourly[meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 6.1 of combined_equipment_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 7: for each virtual meter in list, get energy input data from energy database + #################################################################################################################### + energy_virtual_meter_hourly = dict() + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + try: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_virtual_meter_hourly " + " WHERE virtual_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_virtual_meter_hourly[virtual_meter_id] = None + else: + energy_virtual_meter_hourly[virtual_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 7.1 of combined_equipment_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 8: for each offline meter in list, get energy input data from energy database + #################################################################################################################### + energy_offline_meter_hourly = dict() + if offline_meter_list is not None and len(offline_meter_list) > 0: + try: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_offline_meter_hourly " + " WHERE offline_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_offline_meter_hourly[offline_meter_id] = None + else: + energy_offline_meter_hourly[offline_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1] + + except Exception as e: + error_string = "Error in step 8.1 of combined_equipment_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 9: for each equipment in list, get energy input data from energy database + #################################################################################################################### + energy_equipment_hourly = dict() + if equipment_list is not None and len(equipment_list) > 0: + try: + for equipment in equipment_list: + equipment_id = str(equipment['id']) + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_equipment_input_item_hourly " + " WHERE equipment_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (equipment_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_equipment_hourly[equipment_id] = None + else: + energy_equipment_hourly[equipment_id] = dict() + for row_value in rows_energy_values: + current_datetime_utc = row_value[0] + if current_datetime_utc not in energy_equipment_hourly[equipment_id]: + energy_equipment_hourly[equipment_id][current_datetime_utc] = dict() + energy_category_id = row_value[1] + actual_value = row_value[2] + energy_equipment_hourly[equipment_id][current_datetime_utc][energy_category_id] = \ + actual_value + except Exception as e: + error_string = "Error in step 9 of combined_equipment_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 10: determine common time slot to aggregate + #################################################################################################################### + + common_start_datetime_utc = start_datetime_utc + common_end_datetime_utc = end_datetime_utc + + print("Getting common time slot of energy values for all meters") + if energy_meter_hourly is not None and len(energy_meter_hourly) > 0: + for meter_id, energy_hourly in energy_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all virtual meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0: + for meter_id, energy_hourly in energy_virtual_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all offline meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0: + for meter_id, energy_hourly in energy_offline_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all equipments") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_equipment_hourly is not None and len(energy_equipment_hourly) > 0: + for equipment_id, energy_hourly in energy_equipment_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \ + (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \ + (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0) and \ + (energy_equipment_hourly is None or len(energy_equipment_hourly) == 0): + # There isn't any energy data + print("There isn't any energy data") + # continue the for combined equipment loop to the next combined equipment + print("continue the for combined equipment loop to the next combined equipment") + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + return None + + print("common_start_datetime_utc: " + str(common_start_datetime_utc)) + print("common_end_datetime_utc: " + str(common_end_datetime_utc)) + + #################################################################################################################### + # Step 11: aggregate energy data in the common time slot by energy items and hourly + #################################################################################################################### + + print("Step 11: aggregate energy data in the common time slot by energy items and hourly") + aggregated_values = list() + try: + current_datetime_utc = common_start_datetime_utc + while common_start_datetime_utc is not None \ + and common_end_datetime_utc is not None \ + and current_datetime_utc <= common_end_datetime_utc: + aggregated_value = dict() + aggregated_value['start_datetime_utc'] = current_datetime_utc + aggregated_value['meta_data'] = dict() + + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + energy_item_id = meter['energy_item_id'] + actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + energy_item_id = virtual_meter['energy_item_id'] + actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if offline_meter_list is not None and len(offline_meter_list) > 0: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + energy_item_id = offline_meter['energy_item_id'] + actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if equipment_list is not None and len(equipment_list) > 0: + for equipment in equipment_list: + equipment_id = str(equipment['id']) + meta_data_dict = energy_equipment_hourly[equipment_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_category_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + aggregated_values.append(aggregated_value) + + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + except Exception as e: + error_string = "Error in step 11 of combined_equipment_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 12: save energy data to energy database + #################################################################################################################### + print("Step 12: save energy data to energy database") + + if len(aggregated_values) > 0: + try: + add_values = (" INSERT INTO tbl_combined_equipment_input_item_hourly " + " (combined_equipment_id, " + " energy_item_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for aggregated_value in aggregated_values: + for energy_item_id, actual_value in aggregated_value['meta_data'].items(): + add_values += " (" + str(combined_equipment['id']) + "," + add_values += " " + str(energy_item_id) + "," + add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "'," + add_values += str(actual_value) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_energy_db.execute(add_values[:-2]) + cnx_energy_db.commit() + + except Exception as e: + error_string = "Error in step 12.1 of combined_equipment_energy_input_item.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + else: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() diff --git a/myems-aggregation/combined_equipment_energy_output_category.py b/myems-aggregation/combined_equipment_energy_output_category.py new file mode 100644 index 00000000..d3aa50ff --- /dev/null +++ b/myems-aggregation/combined_equipment_energy_output_category.py @@ -0,0 +1,619 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +from multiprocessing import Pool +import random +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all combined equipments +# Step 2: Create multiprocessing pool to call worker in parallel +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all combined equipments + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of combined_equipment_energy_output_category.main " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_combined_equipments " + " ORDER BY id ") + rows_combined_equipments = cursor_system_db.fetchall() + + if rows_combined_equipments is None or len(rows_combined_equipments) == 0: + print("There isn't any combined equipments ") + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + + combined_equipment_list = list() + for row in rows_combined_equipments: + combined_equipment_list.append({"id": row[0], "name": row[1]}) + + except Exception as e: + logger.error("Error in step 1.2 of combined_equipment_energy_output_category.main " + str(e)) + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + print("Got all combined equipments in MyEMS System Database") + + # shuffle the combined equipment list for randomly calculating the meter hourly value + random.shuffle(combined_equipment_list) + + ################################################################################################################ + # Step 2: Create multiprocessing pool to call worker in parallel + ################################################################################################################ + p = Pool(processes=config.pool_size) + error_list = p.map(worker, combined_equipment_list) + p.close() + p.join() + + for error in error_list: + if error is not None and len(error) > 0: + logger.error(error) + + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of outer while + + +######################################################################################################################## +# PROCEDURES: +# Step 1: get all output meters associated with the combined equipment +# Step 2: get all output virtual meters associated with the combined equipment +# Step 3: get all output offline meters associated with the combined equipment +# Step 4: get all equipments associated with the combined equipment +# Step 5: determine start datetime and end datetime to aggregate +# Step 6: for each meter in list, get energy output data from energy database +# Step 7: for each virtual meter in list, get energy output data from energy database +# Step 8: for each offline meter in list, get energy output data from energy database +# Step 9: for each equipment in list, get energy output data from energy database +# Step 10: determine common time slot to aggregate +# Step 11: aggregate energy data in the common time slot by energy categories and hourly +# Step 12: save energy data to energy database +# +# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter +######################################################################################################################## + +def worker(combined_equipment): + #################################################################################################################### + # Step 1: get all output meters associated with the combined equipment + #################################################################################################################### + print("Step 1: get all output meters associated with the combined equipment " + str(combined_equipment['name'])) + + meter_list = list() + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + error_string = "Error in step 1.1 of combined_equipment_energy_output_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_meters m, tbl_combined_equipments_meters em " + " WHERE m.id = em.meter_id " + " AND m.is_counted = true " + " AND em.is_output = true " + " AND em.combined_equipment_id = %s ", + (combined_equipment['id'],)) + rows_meters = cursor_system_db.fetchall() + + if rows_meters is not None and len(rows_meters) > 0: + for row in rows_meters: + meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 1.2 of combined_equipment_energy_output_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 2: get all output virtual meters associated with the combined equipment + #################################################################################################################### + print("Step 2: get all output virtual meters associated with the combined equipment") + virtual_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_virtual_meters m, tbl_combined_equipments_virtual_meters em " + " WHERE m.id = em.virtual_meter_id " + " AND m.is_counted = true " + " AND em.is_output = true " + " AND em.combined_equipment_id = %s ", + (combined_equipment['id'],)) + rows_virtual_meters = cursor_system_db.fetchall() + + if rows_virtual_meters is not None and len(rows_virtual_meters) > 0: + for row in rows_virtual_meters: + virtual_meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 2.1 of combined_equipment_energy_output_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 3: get all output offline meters associated with the combined equipment + #################################################################################################################### + print("Step 3: get all output offline meters associated with the combined equipment") + + offline_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_offline_meters m, tbl_combined_equipments_offline_meters em " + " WHERE m.id = em.offline_meter_id " + " AND m.is_counted = true " + " AND em.is_output = true " + " AND em.combined_equipment_id = %s ", + (combined_equipment['id'],)) + rows_offline_meters = cursor_system_db.fetchall() + + if rows_offline_meters is not None and len(rows_offline_meters) > 0: + for row in rows_offline_meters: + offline_meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 3.1 of combined_equipment_energy_output_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + #################################################################################################################### + # Step 4: get all equipments associated with the combined equipment + #################################################################################################################### + print("Step 4: get all equipments associated with the combined equipment") + + equipment_list = list() + + try: + cursor_system_db.execute(" SELECT e.id, e.name " + " FROM tbl_equipments e, tbl_combined_equipments_equipments ce " + " WHERE e.id = ce.equipment_id " + " AND e.is_output_counted = true " + " AND ce.combined_equipment_id = %s ", + (combined_equipment['id'],)) + rows_equipments = cursor_system_db.fetchall() + + if rows_equipments is not None and len(rows_equipments) > 0: + for row in rows_equipments: + equipment_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 4 of combined_equipment_energy_output_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + #################################################################################################################### + # stop to the next combined equipment if this combined 3equipment is empty + #################################################################################################################### + if (meter_list is None or len(meter_list) == 0) and \ + (virtual_meter_list is None or len(virtual_meter_list) == 0) and \ + (offline_meter_list is None or len(offline_meter_list) == 0) and \ + (equipment_list is None or len(equipment_list) == 0): + print("This is an empty combined equipment ") + return None + + #################################################################################################################### + # Step 5: determine start datetime and end datetime to aggregate + #################################################################################################################### + print("Step 5: determine start datetime and end datetime to aggregate") + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + error_string = "Error in step 5.1 of combined_equipment_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + try: + query = (" SELECT MAX(start_datetime_utc) " + " FROM tbl_combined_equipment_output_category_hourly " + " WHERE combined_equipment_id = %s ") + cursor_energy_db.execute(query, (combined_equipment['id'],)) + row_datetime = cursor_energy_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19] + + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19]) + + except Exception as e: + error_string = "Error in step 5.2 of combined_equipment_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 6: for each meter in list, get energy output data from energy database + #################################################################################################################### + energy_meter_hourly = dict() + try: + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_meter_hourly " + " WHERE meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_meter_hourly[meter_id] = None + else: + energy_meter_hourly[meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 6.1 of combined_equipment_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 7: for each virtual meter in list, get energy output data from energy database + #################################################################################################################### + energy_virtual_meter_hourly = dict() + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + try: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_virtual_meter_hourly " + " WHERE virtual_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_virtual_meter_hourly[virtual_meter_id] = None + else: + energy_virtual_meter_hourly[virtual_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 7.1 of combined_equipment_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 8: for each offline meter in list, get energy output data from energy database + #################################################################################################################### + energy_offline_meter_hourly = dict() + if offline_meter_list is not None and len(offline_meter_list) > 0: + try: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_offline_meter_hourly " + " WHERE offline_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_offline_meter_hourly[offline_meter_id] = None + else: + energy_offline_meter_hourly[offline_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1] + + except Exception as e: + error_string = "Error in step 8.1 of combined_equipment_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 9: for each equipment in list, get energy output data from energy database + #################################################################################################################### + energy_equipment_hourly = dict() + if equipment_list is not None and len(equipment_list) > 0: + try: + for equipment in equipment_list: + equipment_id = str(equipment['id']) + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_equipment_output_category_hourly " + " WHERE equipment_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (equipment_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_equipment_hourly[equipment_id] = None + else: + energy_equipment_hourly[equipment_id] = dict() + for row_value in rows_energy_values: + current_datetime_utc = row_value[0] + if current_datetime_utc not in energy_equipment_hourly[equipment_id]: + energy_equipment_hourly[equipment_id][current_datetime_utc] = dict() + energy_category_id = row_value[1] + actual_value = row_value[2] + energy_equipment_hourly[equipment_id][current_datetime_utc][energy_category_id] = \ + actual_value + except Exception as e: + error_string = "Error in step 9 of combined_equipment_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 10: determine common time slot to aggregate + #################################################################################################################### + + common_start_datetime_utc = start_datetime_utc + common_end_datetime_utc = end_datetime_utc + + print("Getting common time slot of energy values for all meters") + if energy_meter_hourly is not None and len(energy_meter_hourly) > 0: + for meter_id, energy_hourly in energy_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all virtual meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0: + for meter_id, energy_hourly in energy_virtual_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all offline meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0: + for meter_id, energy_hourly in energy_offline_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all equipments...") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_equipment_hourly is not None and len(energy_equipment_hourly) > 0: + for equipment_id, energy_hourly in energy_equipment_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \ + (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \ + (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0) and \ + (energy_equipment_hourly is None or len(energy_equipment_hourly) == 0): + # There isn't any energy data + print("There isn't any energy data") + # continue the for combined equipment loop to the next combined equipment + print("continue the for combined equipment loop to the next combined equipment") + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + return None + + print("common_start_datetime_utc: " + str(common_start_datetime_utc)) + print("common_end_datetime_utc: " + str(common_end_datetime_utc)) + + #################################################################################################################### + # Step 11: aggregate energy data in the common time slot by energy categories and hourly + #################################################################################################################### + + print("Step 11: aggregate energy data in the common time slot by energy categories and hourly") + aggregated_values = list() + try: + current_datetime_utc = common_start_datetime_utc + while common_start_datetime_utc is not None \ + and common_end_datetime_utc is not None \ + and current_datetime_utc <= common_end_datetime_utc: + aggregated_value = dict() + aggregated_value['start_datetime_utc'] = current_datetime_utc + aggregated_value['meta_data'] = dict() + + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + energy_category_id = meter['energy_category_id'] + actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + energy_category_id = virtual_meter['energy_category_id'] + actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if offline_meter_list is not None and len(offline_meter_list) > 0: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + energy_category_id = offline_meter['energy_category_id'] + actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if equipment_list is not None and len(equipment_list) > 0: + for equipment in equipment_list: + equipment_id = str(equipment['id']) + meta_data_dict = energy_equipment_hourly[equipment_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_category_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + aggregated_values.append(aggregated_value) + + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + except Exception as e: + error_string = "Error in step 11 of combined_equipment_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 12: save energy data to energy database + #################################################################################################################### + print("Step 12: save energy data to energy database") + + if len(aggregated_values) > 0: + try: + add_values = (" INSERT INTO tbl_combined_equipment_output_category_hourly " + " (combined_equipment_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for aggregated_value in aggregated_values: + for energy_category_id, actual_value in aggregated_value['meta_data'].items(): + add_values += " (" + str(combined_equipment['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "'," + add_values += str(actual_value) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_energy_db.execute(add_values[:-2]) + cnx_energy_db.commit() + + except Exception as e: + error_string = "Error in step 12.1 of combined_equipment_energy_output_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + else: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() diff --git a/myems-aggregation/config.py b/myems-aggregation/config.py new file mode 100644 index 00000000..cddd91e1 --- /dev/null +++ b/myems-aggregation/config.py @@ -0,0 +1,45 @@ +myems_system_db = { + 'user': 'root', + 'password': '!MyEMS1', + 'host': '127.0.0.1', + 'database': 'myems_system_db', +} + +myems_historical_db = { + 'user': 'root', + 'password': '!MyEMS1', + 'host': '127.0.0.1', + 'database': 'myems_historical_db', +} + +myems_energy_db = { + 'user': 'root', + 'password': '!MyEMS1', + 'host': '127.0.0.1', + 'database': 'myems_energy_db', +} + +myems_billing_db = { + 'user': 'root', + 'password': '!MyEMS1', + 'host': '127.0.0.1', + 'database': 'myems_billing_db', +} + + +# indicates how long in minutes energy data will be aggregated +# 30 for half hourly +# 60 for hourly +minutes_to_count = 60 + +# indicates from when (in UTC timezone) to recalculate if the energy data is null or were cleared +# format string: '%Y-%m-%d %H:%M:%S' +start_datetime_utc = '2019-12-31 16:00:00' + +# indicates the project's time zone offset from UTC +utc_offset = '+08:00' + +# the number of worker processes in parallel +# the pool size depends on the computing performance of the database server and the analysis server +pool_size = 5 + diff --git a/myems-aggregation/equipment_billing_input_category.py b/myems-aggregation/equipment_billing_input_category.py new file mode 100644 index 00000000..5b2bfc81 --- /dev/null +++ b/myems-aggregation/equipment_billing_input_category.py @@ -0,0 +1,269 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +import tariff +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all equipments +# for each equipment in list: +# Step 2: get the latest start_datetime_utc +# Step 3: get all energy input data since the latest start_datetime_utc +# Step 4: get tariffs +# Step 5: calculate billing by multiplying energy with tariff +# Step 6: save billing data to database +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all equipments + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of equipment_billing_input_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name, cost_center_id " + " FROM tbl_equipments " + " ORDER BY id ") + rows_equipments = cursor_system_db.fetchall() + + if rows_equipments is None or len(rows_equipments) == 0: + print("Step 1.2: There isn't any equipments. ") + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + equipment_list = list() + for row in rows_equipments: + equipment_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]}) + + except Exception as e: + logger.error("Error in step 1.2 of equipment_billing_input_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Step 1.2: Got all equipments from MyEMS System Database") + + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + logger.error("Error in step 1.3 of equipment_billing_input_category " + str(e)) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Energy Database") + + cnx_billing_db = None + cursor_billing_db = None + try: + cnx_billing_db = mysql.connector.connect(**config.myems_billing_db) + cursor_billing_db = cnx_billing_db.cursor() + except Exception as e: + logger.error("Error in step 1.4 of equipment_billing_input_category " + str(e)) + if cursor_billing_db: + cursor_billing_db.close() + if cnx_billing_db: + cnx_billing_db.close() + + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Billing Database") + + for equipment in equipment_list: + + ############################################################################################################ + # Step 2: get the latest start_datetime_utc + ############################################################################################################ + print("Step 2: get the latest start_datetime_utc from billing database for " + equipment['name']) + try: + cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) " + " FROM tbl_equipment_input_category_hourly " + " WHERE equipment_id = %s ", + (equipment['id'], )) + row_datetime = cursor_billing_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]) + except Exception as e: + logger.error("Error in step 2 of equipment_billing_input_category " + str(e)) + # break the for equipment loop + break + + ############################################################################################################ + # Step 3: get all energy input data since the latest start_datetime_utc + ############################################################################################################ + print("Step 3: get all energy input data since the latest start_datetime_utc") + + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_equipment_input_category_hourly " + " WHERE equipment_id = %s AND start_datetime_utc >= %s " + " ORDER BY id ") + cursor_energy_db.execute(query, (equipment['id'], start_datetime_utc, )) + rows_hourly = cursor_energy_db.fetchall() + + if rows_hourly is None or len(rows_hourly) == 0: + print("Step 3: There isn't any energy input data to calculate. ") + # continue the for equipment loop + continue + + energy_dict = dict() + energy_category_list = list() + end_datetime_utc = start_datetime_utc + for row_hourly in rows_hourly: + current_datetime_utc = row_hourly[0] + energy_category_id = row_hourly[1] + + if energy_category_id not in energy_category_list: + energy_category_list.append(energy_category_id) + + actual_value = row_hourly[2] + if energy_dict.get(current_datetime_utc) is None: + energy_dict[current_datetime_utc] = dict() + energy_dict[current_datetime_utc][energy_category_id] = actual_value + if current_datetime_utc > end_datetime_utc: + end_datetime_utc = current_datetime_utc + + ############################################################################################################ + # Step 4: get tariffs + ############################################################################################################ + print("Step 4: get tariffs") + tariff_dict = dict() + for energy_category_id in energy_category_list: + tariff_dict[energy_category_id] = tariff.get_energy_category_tariffs(equipment['cost_center_id'], + energy_category_id, + start_datetime_utc, + end_datetime_utc) + ############################################################################################################ + # Step 5: calculate billing by multiplying energy with tariff + ############################################################################################################ + print("Step 5: calculate billing by multiplying energy with tariff") + billing_dict = dict() + + if len(energy_dict) > 0: + for current_datetime_utc in energy_dict.keys(): + billing_dict[current_datetime_utc] = dict() + for energy_category_id in energy_category_list: + current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc) + current_energy = energy_dict[current_datetime_utc].get(energy_category_id) + if current_tariff is not None \ + and isinstance(current_tariff, Decimal) \ + and current_energy is not None \ + and isinstance(current_energy, Decimal): + billing_dict[current_datetime_utc][energy_category_id] = \ + current_energy * current_tariff + + if len(billing_dict[current_datetime_utc]) == 0: + del billing_dict[current_datetime_utc] + + ############################################################################################################ + # Step 6: save billing data to billing database + ############################################################################################################ + print("Step 6: save billing data to billing database") + + if len(billing_dict) > 0: + try: + add_values = (" INSERT INTO tbl_equipment_input_category_hourly " + " (equipment_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for current_datetime_utc in billing_dict: + for energy_category_id in energy_category_list: + current_billing = billing_dict[current_datetime_utc].get(energy_category_id) + if current_billing is not None and isinstance(current_billing, Decimal): + add_values += " (" + str(equipment['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + current_datetime_utc.isoformat()[0:19] + "'," + add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_billing_db.execute(add_values[:-2]) + cnx_billing_db.commit() + except Exception as e: + logger.error("Error in step 6 of equipment_billing_input_category " + str(e)) + # break the for equipment loop + break + + # end of for equipment loop + if cnx_system_db: + cnx_system_db.close() + if cursor_system_db: + cursor_system_db.close() + + if cnx_energy_db: + cnx_energy_db.close() + if cursor_energy_db: + cursor_energy_db.close() + + if cnx_billing_db: + cnx_billing_db.close() + if cursor_billing_db: + cursor_billing_db.close() + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of the outermost while loop diff --git a/myems-aggregation/equipment_billing_input_item.py b/myems-aggregation/equipment_billing_input_item.py new file mode 100644 index 00000000..ea094f32 --- /dev/null +++ b/myems-aggregation/equipment_billing_input_item.py @@ -0,0 +1,269 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +import tariff +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all equipments +# for each equipment in list: +# Step 2: get the latest start_datetime_utc +# Step 3: get all energy input data since the latest start_datetime_utc +# Step 4: get tariffs +# Step 5: calculate billing by multiplying energy with tariff +# Step 6: save billing data to database +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all equipments + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of equipment_billing_input_item " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name, cost_center_id " + " FROM tbl_equipments " + " ORDER BY id ") + rows_equipments = cursor_system_db.fetchall() + + if rows_equipments is None or len(rows_equipments) == 0: + print("Step 1.2: There isn't any equipments. ") + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + equipment_list = list() + for row in rows_equipments: + equipment_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]}) + + except Exception as e: + logger.error("Error in step 1.2 of equipment_billing_input_item " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Step 1.2: Got all equipments from MyEMS System Database") + + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + logger.error("Error in step 1.3 of equipment_billing_input_item " + str(e)) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Energy Database") + + cnx_billing_db = None + cursor_billing_db = None + try: + cnx_billing_db = mysql.connector.connect(**config.myems_billing_db) + cursor_billing_db = cnx_billing_db.cursor() + except Exception as e: + logger.error("Error in step 1.4 of equipment_billing_input_item " + str(e)) + if cursor_billing_db: + cursor_billing_db.close() + if cnx_billing_db: + cnx_billing_db.close() + + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Billing Database") + + for equipment in equipment_list: + + ############################################################################################################ + # Step 2: get the latest start_datetime_utc + ############################################################################################################ + print("Step 2: get the latest start_datetime_utc from billing database for " + equipment['name']) + try: + cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) " + " FROM tbl_equipment_input_item_hourly " + " WHERE equipment_id = %s ", + (equipment['id'], )) + row_datetime = cursor_billing_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]) + except Exception as e: + logger.error("Error in step 2 of equipment_billing_input_item " + str(e)) + # break the for equipment loop + break + + ############################################################################################################ + # Step 3: get all energy input data since the latest start_datetime_utc + ############################################################################################################ + print("Step 3: get all energy input data since the latest start_datetime_utc") + + query = (" SELECT start_datetime_utc, energy_item_id, actual_value " + " FROM tbl_equipment_input_item_hourly " + " WHERE equipment_id = %s AND start_datetime_utc >= %s " + " ORDER BY id ") + cursor_energy_db.execute(query, (equipment['id'], start_datetime_utc, )) + rows_hourly = cursor_energy_db.fetchall() + + if rows_hourly is None or len(rows_hourly) == 0: + print("Step 3: There isn't any energy input data to calculate. ") + # continue the for equipment loop + continue + + energy_dict = dict() + energy_item_list = list() + end_datetime_utc = start_datetime_utc + for row_hourly in rows_hourly: + current_datetime_utc = row_hourly[0] + energy_item_id = row_hourly[1] + + if energy_item_id not in energy_item_list: + energy_item_list.append(energy_item_id) + + actual_value = row_hourly[2] + if energy_dict.get(current_datetime_utc) is None: + energy_dict[current_datetime_utc] = dict() + energy_dict[current_datetime_utc][energy_item_id] = actual_value + if current_datetime_utc > end_datetime_utc: + end_datetime_utc = current_datetime_utc + + ############################################################################################################ + # Step 4: get tariffs + ############################################################################################################ + print("Step 4: get tariffs") + tariff_dict = dict() + for energy_item_id in energy_item_list: + tariff_dict[energy_item_id] = tariff.get_energy_item_tariffs(equipment['cost_center_id'], + energy_item_id, + start_datetime_utc, + end_datetime_utc) + ############################################################################################################ + # Step 5: calculate billing by multiplying energy with tariff + ############################################################################################################ + print("Step 5: calculate billing by multiplying energy with tariff") + billing_dict = dict() + + if len(energy_dict) > 0: + for current_datetime_utc in energy_dict.keys(): + billing_dict[current_datetime_utc] = dict() + for energy_item_id in energy_item_list: + current_tariff = tariff_dict[energy_item_id].get(current_datetime_utc) + current_energy = energy_dict[current_datetime_utc].get(energy_item_id) + if current_tariff is not None \ + and isinstance(current_tariff, Decimal) \ + and current_energy is not None \ + and isinstance(current_energy, Decimal): + billing_dict[current_datetime_utc][energy_item_id] = \ + current_energy * current_tariff + + if len(billing_dict[current_datetime_utc]) == 0: + del billing_dict[current_datetime_utc] + + ############################################################################################################ + # Step 6: save billing data to billing database + ############################################################################################################ + print("Step 6: save billing data to billing database") + + if len(billing_dict) > 0: + try: + add_values = (" INSERT INTO tbl_equipment_input_item_hourly " + " (equipment_id, " + " energy_item_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for current_datetime_utc in billing_dict: + for energy_item_id in energy_item_list: + current_billing = billing_dict[current_datetime_utc].get(energy_item_id) + if current_billing is not None and isinstance(current_billing, Decimal): + add_values += " (" + str(equipment['id']) + "," + add_values += " " + str(energy_item_id) + "," + add_values += "'" + current_datetime_utc.isoformat()[0:19] + "'," + add_values += str(billing_dict[current_datetime_utc][energy_item_id]) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_billing_db.execute(add_values[:-2]) + cnx_billing_db.commit() + except Exception as e: + logger.error("Error in step 6 of equipment_billing_input_item " + str(e)) + # break the for equipment loop + break + + # end of for equipment loop + if cnx_system_db: + cnx_system_db.close() + if cursor_system_db: + cursor_system_db.close() + + if cnx_energy_db: + cnx_energy_db.close() + if cursor_energy_db: + cursor_energy_db.close() + + if cnx_billing_db: + cnx_billing_db.close() + if cursor_billing_db: + cursor_billing_db.close() + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of the outermost while loop diff --git a/myems-aggregation/equipment_billing_output_category.py b/myems-aggregation/equipment_billing_output_category.py new file mode 100644 index 00000000..76668ebe --- /dev/null +++ b/myems-aggregation/equipment_billing_output_category.py @@ -0,0 +1,269 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +import tariff +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all equipments +# for each equipment in list: +# Step 2: get the latest start_datetime_utc +# Step 3: get all energy input data since the latest start_datetime_utc +# Step 4: get tariffs +# Step 5: calculate billing by multiplying energy with tariff +# Step 6: save billing data to database +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all equipments + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of equipment_billing_input_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name, cost_center_id " + " FROM tbl_equipments " + " ORDER BY id ") + rows_equipments = cursor_system_db.fetchall() + + if rows_equipments is None or len(rows_equipments) == 0: + print("Step 1.2: There isn't any equipments. ") + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + equipment_list = list() + for row in rows_equipments: + equipment_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]}) + + except Exception as e: + logger.error("Error in step 1.2 of equipment_billing_output_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Step 1.2: Got all equipments from MyEMS System Database") + + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + logger.error("Error in step 1.3 of equipment_billing_output_category " + str(e)) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Energy Database") + + cnx_billing_db = None + cursor_billing_db = None + try: + cnx_billing_db = mysql.connector.connect(**config.myems_billing_db) + cursor_billing_db = cnx_billing_db.cursor() + except Exception as e: + logger.error("Error in step 1.4 of equipment_billing_output_category " + str(e)) + if cursor_billing_db: + cursor_billing_db.close() + if cnx_billing_db: + cnx_billing_db.close() + + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Billing Database") + + for equipment in equipment_list: + + ############################################################################################################ + # Step 2: get the latest start_datetime_utc + ############################################################################################################ + print("Step 2: get the latest start_datetime_utc from billing database for " + equipment['name']) + try: + cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) " + " FROM tbl_equipment_output_category_hourly " + " WHERE equipment_id = %s ", + (equipment['id'], )) + row_datetime = cursor_billing_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]) + except Exception as e: + logger.error("Error in step 2 of equipment_billing_output_category " + str(e)) + # break the for equipment loop + break + + ############################################################################################################ + # Step 3: get all energy output data since the latest start_datetime_utc + ############################################################################################################ + print("Step 3: get all energy output data since the latest start_datetime_utc") + + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_equipment_output_category_hourly " + " WHERE equipment_id = %s AND start_datetime_utc >= %s " + " ORDER BY id ") + cursor_energy_db.execute(query, (equipment['id'], start_datetime_utc, )) + rows_hourly = cursor_energy_db.fetchall() + + if rows_hourly is None or len(rows_hourly) == 0: + print("Step 3: There isn't any energy output data to calculate. ") + # continue the for equipment loop + continue + + energy_dict = dict() + energy_category_list = list() + end_datetime_utc = start_datetime_utc + for row_hourly in rows_hourly: + current_datetime_utc = row_hourly[0] + energy_category_id = row_hourly[1] + + if energy_category_id not in energy_category_list: + energy_category_list.append(energy_category_id) + + actual_value = row_hourly[2] + if energy_dict.get(current_datetime_utc) is None: + energy_dict[current_datetime_utc] = dict() + energy_dict[current_datetime_utc][energy_category_id] = actual_value + if current_datetime_utc > end_datetime_utc: + end_datetime_utc = current_datetime_utc + + ############################################################################################################ + # Step 4: get tariffs + ############################################################################################################ + print("Step 4: get tariffs") + tariff_dict = dict() + for energy_category_id in energy_category_list: + tariff_dict[energy_category_id] = tariff.get_energy_category_tariffs(equipment['cost_center_id'], + energy_category_id, + start_datetime_utc, + end_datetime_utc) + ############################################################################################################ + # Step 5: calculate billing by multiplying energy with tariff + ############################################################################################################ + print("Step 5: calculate billing by multiplying energy with tariff") + billing_dict = dict() + + if len(energy_dict) > 0: + for current_datetime_utc in energy_dict.keys(): + billing_dict[current_datetime_utc] = dict() + for energy_category_id in energy_category_list: + current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc) + current_energy = energy_dict[current_datetime_utc].get(energy_category_id) + if current_tariff is not None \ + and isinstance(current_tariff, Decimal) \ + and current_energy is not None \ + and isinstance(current_energy, Decimal): + billing_dict[current_datetime_utc][energy_category_id] = \ + current_energy * current_tariff + + if len(billing_dict[current_datetime_utc]) == 0: + del billing_dict[current_datetime_utc] + + ############################################################################################################ + # Step 6: save billing data to billing database + ############################################################################################################ + print("Step 6: save billing data to billing database") + + if len(billing_dict) > 0: + try: + add_values = (" INSERT INTO tbl_equipment_output_category_hourly " + " (equipment_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for current_datetime_utc in billing_dict: + for energy_category_id in energy_category_list: + current_billing = billing_dict[current_datetime_utc].get(energy_category_id) + if current_billing is not None and isinstance(current_billing, Decimal): + add_values += " (" + str(equipment['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + current_datetime_utc.isoformat()[0:19] + "'," + add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_billing_db.execute(add_values[:-2]) + cnx_billing_db.commit() + except Exception as e: + logger.error("Error in step 6 of equipment_billing_output_category " + str(e)) + # break the for equipment loop + break + + # end of for equipment loop + if cnx_system_db: + cnx_system_db.close() + if cursor_system_db: + cursor_system_db.close() + + if cnx_energy_db: + cnx_energy_db.close() + if cursor_energy_db: + cursor_energy_db.close() + + if cnx_billing_db: + cnx_billing_db.close() + if cursor_billing_db: + cursor_billing_db.close() + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of the outermost while loop diff --git a/myems-aggregation/equipment_energy_input_category.py b/myems-aggregation/equipment_energy_input_category.py new file mode 100644 index 00000000..e672e688 --- /dev/null +++ b/myems-aggregation/equipment_energy_input_category.py @@ -0,0 +1,524 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +from multiprocessing import Pool +import random +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all equipments +# Step 2: Create multiprocessing pool to call worker in parallel +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all equipments + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of equipment_energy_input_category.main " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_equipments " + " ORDER BY id ") + rows_equipments = cursor_system_db.fetchall() + + if rows_equipments is None or len(rows_equipments) == 0: + print("There isn't any equipments ") + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + + equipment_list = list() + for row in rows_equipments: + equipment_list.append({"id": row[0], "name": row[1]}) + + except Exception as e: + logger.error("Error in step 1.2 of equipment_energy_input_category.main " + str(e)) + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + print("Got all equipments in MyEMS System Database") + + # shuffle the equipment list for randomly calculating the meter hourly value + random.shuffle(equipment_list) + + ################################################################################################################ + # Step 2: Create multiprocessing pool to call worker in parallel + ################################################################################################################ + p = Pool(processes=config.pool_size) + error_list = p.map(worker, equipment_list) + p.close() + p.join() + + for error in error_list: + if error is not None and len(error) > 0: + logger.error(error) + + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of outer while + + +######################################################################################################################## +# PROCEDURES: +# Step 1: get all input meters associated with the equipment +# Step 2: get all input virtual meters associated with the equipment +# Step 3: get all input offline meters associated with the equipment +# Step 4: determine start datetime and end datetime to aggregate +# Step 5: for each meter in list, get energy input data from energy database +# Step 6: for each virtual meter in list, get energy input data from energy database +# Step 7: for each offline meter in list, get energy input data from energy database +# Step 8: determine common time slot to aggregate +# Step 9: aggregate energy data in the common time slot by energy categories and hourly +# Step 10: save energy data to energy database +# +# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter +######################################################################################################################## + +def worker(equipment): + #################################################################################################################### + # Step 1: get all input meters associated with the equipment + #################################################################################################################### + print("Step 1: get all input meters associated with the equipment " + str(equipment['name'])) + + meter_list = list() + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + error_string = "Error in step 1.1 of equipment_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_meters m, tbl_equipments_meters em " + " WHERE m.id = em.meter_id " + " AND m.is_counted = true " + " AND em.is_output = false " + " AND em.equipment_id = %s ", + (equipment['id'],)) + rows_meters = cursor_system_db.fetchall() + + if rows_meters is not None and len(rows_meters) > 0: + for row in rows_meters: + meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 1.2 of equipment_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 2: get all input virtual meters associated with the equipment + #################################################################################################################### + print("Step 2: get all input virtual meters associated with the equipment") + virtual_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_virtual_meters m, tbl_equipments_virtual_meters em " + " WHERE m.id = em.virtual_meter_id " + " AND m.is_counted = true " + " AND em.is_output = false " + " AND em.equipment_id = %s ", + (equipment['id'],)) + rows_virtual_meters = cursor_system_db.fetchall() + + if rows_virtual_meters is not None and len(rows_virtual_meters) > 0: + for row in rows_virtual_meters: + virtual_meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 2.1 of equipment_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 3: get all input offline meters associated with the equipment + #################################################################################################################### + print("Step 3: get all input offline meters associated with the equipment") + + offline_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_offline_meters m, tbl_equipments_offline_meters em " + " WHERE m.id = em.offline_meter_id " + " AND m.is_counted = true " + " AND em.is_output = false " + " AND em.equipment_id = %s ", + (equipment['id'],)) + rows_offline_meters = cursor_system_db.fetchall() + + if rows_offline_meters is not None and len(rows_offline_meters) > 0: + for row in rows_offline_meters: + offline_meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 3.1 of equipment_energy_input_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + #################################################################################################################### + # stop to the next equipment if this equipment is empty + #################################################################################################################### + if (meter_list is None or len(meter_list) == 0) and \ + (virtual_meter_list is None or len(virtual_meter_list) == 0) and \ + (offline_meter_list is None or len(offline_meter_list) == 0): + print("This is an empty equipment ") + return None + + #################################################################################################################### + # Step 4: determine start datetime and end datetime to aggregate + #################################################################################################################### + print("Step 4: determine start datetime and end datetime to aggregate") + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + error_string = "Error in step 4.1 of equipment_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + try: + query = (" SELECT MAX(start_datetime_utc) " + " FROM tbl_equipment_input_category_hourly " + " WHERE equipment_id = %s ") + cursor_energy_db.execute(query, (equipment['id'],)) + row_datetime = cursor_energy_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19] + + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19]) + + except Exception as e: + error_string = "Error in step 4.2 of equipment_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 5: for each meter in list, get energy input data from energy database + #################################################################################################################### + energy_meter_hourly = dict() + try: + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_meter_hourly " + " WHERE meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_meter_hourly[meter_id] = None + else: + energy_meter_hourly[meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 5.1 of equipment_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 6: for each virtual meter in list, get energy input data from energy database + #################################################################################################################### + energy_virtual_meter_hourly = dict() + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + try: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_virtual_meter_hourly " + " WHERE virtual_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_virtual_meter_hourly[virtual_meter_id] = None + else: + energy_virtual_meter_hourly[virtual_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 6.1 of equipment_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 7: for each offline meter in list, get energy input data from energy database + #################################################################################################################### + energy_offline_meter_hourly = dict() + if offline_meter_list is not None and len(offline_meter_list) > 0: + try: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_offline_meter_hourly " + " WHERE offline_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_offline_meter_hourly[offline_meter_id] = None + else: + energy_offline_meter_hourly[offline_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1] + + except Exception as e: + error_string = "Error in step 7.1 of equipment_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 8: determine common time slot to aggregate + #################################################################################################################### + + common_start_datetime_utc = start_datetime_utc + common_end_datetime_utc = end_datetime_utc + + print("Getting common time slot of energy values for all meters") + if energy_meter_hourly is not None and len(energy_meter_hourly) > 0: + for meter_id, energy_hourly in energy_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all virtual meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0: + for meter_id, energy_hourly in energy_virtual_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all offline meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0: + for meter_id, energy_hourly in energy_offline_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \ + (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \ + (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0): + # There isn't any energy data + print("There isn't any energy data") + # continue the for equipment loop to the next equipment + print("continue the for equipment loop to the next equipment") + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + return None + + print("common_start_datetime_utc: " + str(common_start_datetime_utc)) + print("common_end_datetime_utc: " + str(common_end_datetime_utc)) + + #################################################################################################################### + # Step 9: aggregate energy data in the common time slot by energy categories and hourly + #################################################################################################################### + + print("Step 9: aggregate energy data in the common time slot by energy categories and hourly") + aggregated_values = list() + try: + current_datetime_utc = common_start_datetime_utc + while common_start_datetime_utc is not None \ + and common_end_datetime_utc is not None \ + and current_datetime_utc <= common_end_datetime_utc: + aggregated_value = dict() + aggregated_value['start_datetime_utc'] = current_datetime_utc + aggregated_value['meta_data'] = dict() + + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + energy_category_id = meter['energy_category_id'] + actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + energy_category_id = virtual_meter['energy_category_id'] + actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if offline_meter_list is not None and len(offline_meter_list) > 0: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + energy_category_id = offline_meter['energy_category_id'] + actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + aggregated_values.append(aggregated_value) + + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + except Exception as e: + error_string = "Error in step 9 of equipment_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 10: save energy data to energy database + #################################################################################################################### + print("Step 10: save energy data to energy database") + + if len(aggregated_values) > 0: + try: + add_values = (" INSERT INTO tbl_equipment_input_category_hourly " + " (equipment_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for aggregated_value in aggregated_values: + for energy_category_id, actual_value in aggregated_value['meta_data'].items(): + add_values += " (" + str(equipment['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "'," + add_values += str(actual_value) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_energy_db.execute(add_values[:-2]) + cnx_energy_db.commit() + + except Exception as e: + error_string = "Error in step 10.1 of equipment_energy_input_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + else: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() diff --git a/myems-aggregation/equipment_energy_input_item.py b/myems-aggregation/equipment_energy_input_item.py new file mode 100644 index 00000000..011d0e85 --- /dev/null +++ b/myems-aggregation/equipment_energy_input_item.py @@ -0,0 +1,527 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +from multiprocessing import Pool +import random +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all equipments +# Step 2: Create multiprocessing pool to call worker in parallel +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all equipments + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of equipment_energy_input_item.main " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_equipments " + " ORDER BY id ") + rows_equipments = cursor_system_db.fetchall() + + if rows_equipments is None or len(rows_equipments) == 0: + print("There isn't any equipments ") + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + + equipment_list = list() + for row in rows_equipments: + equipment_list.append({"id": row[0], "name": row[1]}) + + except Exception as e: + logger.error("Error in step 1.2 of equipment_energy_input_item.main " + str(e)) + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + print("Got all equipments in MyEMS System Database") + + # shuffle the equipment list for randomly calculating the meter hourly value + random.shuffle(equipment_list) + + ################################################################################################################ + # Step 2: Create multiprocessing pool to call worker in parallel + ################################################################################################################ + p = Pool(processes=config.pool_size) + error_list = p.map(worker, equipment_list) + p.close() + p.join() + + for error in error_list: + if error is not None and len(error) > 0: + logger.error(error) + + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of outer while + + +######################################################################################################################## +# PROCEDURES: +# Step 1: get all input meters associated with the equipment +# Step 2: get all input virtual meters associated with the equipment +# Step 3: get all input offline meters associated with the equipment +# Step 4: determine start datetime and end datetime to aggregate +# Step 5: for each meter in list, get energy input data from energy database +# Step 6: for each virtual meter in list, get energy input data from energy database +# Step 7: for each offline meter in list, get energy input data from energy database +# Step 8: determine common time slot to aggregate +# Step 9: aggregate energy data in the common time slot by energy items and hourly +# Step 10: save energy data to energy database +# +# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter +######################################################################################################################## + +def worker(equipment): + #################################################################################################################### + # Step 1: get all input meters associated with the equipment + #################################################################################################################### + print("Step 1: get all input meters associated with the equipment " + str(equipment['name'])) + + meter_list = list() + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + error_string = "Error in step 1.1 of equipment_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_meters m, tbl_equipments_meters em " + " WHERE m.id = em.meter_id " + " AND m.is_counted = true " + " AND m.energy_item_id is NOT NULL " + " AND em.is_output = false " + " AND em.equipment_id = %s ", + (equipment['id'],)) + rows_meters = cursor_system_db.fetchall() + + if rows_meters is not None and len(rows_meters) > 0: + for row in rows_meters: + meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 1.2 of equipment_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 2: get all input virtual meters associated with the equipment + #################################################################################################################### + print("Step 2: get all input virtual meters associated with the equipment") + virtual_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_virtual_meters m, tbl_equipments_virtual_meters em " + " WHERE m.id = em.virtual_meter_id " + " AND m.energy_item_id is NOT NULL " + " AND m.is_counted = true " + " AND em.is_output = false " + " AND em.equipment_id = %s ", + (equipment['id'],)) + rows_virtual_meters = cursor_system_db.fetchall() + + if rows_virtual_meters is not None and len(rows_virtual_meters) > 0: + for row in rows_virtual_meters: + virtual_meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 2.1 of equipment_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 3: get all input offline meters associated with the equipment + #################################################################################################################### + print("Step 3: get all input offline meters associated with the equipment") + + offline_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_offline_meters m, tbl_equipments_offline_meters em " + " WHERE m.id = em.offline_meter_id " + " AND m.energy_item_id is NOT NULL " + " AND m.is_counted = true " + " AND em.is_output = false " + " AND em.equipment_id = %s ", + (equipment['id'],)) + rows_offline_meters = cursor_system_db.fetchall() + + if rows_offline_meters is not None and len(rows_offline_meters) > 0: + for row in rows_offline_meters: + offline_meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 3.1 of equipment_energy_input_item.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + #################################################################################################################### + # stop to the next equipment if this equipment is empty + #################################################################################################################### + if (meter_list is None or len(meter_list) == 0) and \ + (virtual_meter_list is None or len(virtual_meter_list) == 0) and \ + (offline_meter_list is None or len(offline_meter_list) == 0): + print("This is an empty equipment ") + return None + + #################################################################################################################### + # Step 4: determine start datetime and end datetime to aggregate + #################################################################################################################### + print("Step 4: determine start datetime and end datetime to aggregate") + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + error_string = "Error in step 4.1 of equipment_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + try: + query = (" SELECT MAX(start_datetime_utc) " + " FROM tbl_equipment_input_item_hourly " + " WHERE equipment_id = %s ") + cursor_energy_db.execute(query, (equipment['id'],)) + row_datetime = cursor_energy_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19] + + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19]) + + except Exception as e: + error_string = "Error in step 4.2 of equipment_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 5: for each meter in list, get energy input data from energy database + #################################################################################################################### + energy_meter_hourly = dict() + try: + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_meter_hourly " + " WHERE meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_meter_hourly[meter_id] = None + else: + energy_meter_hourly[meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 5.1 of equipment_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 6: for each virtual meter in list, get energy input data from energy database + #################################################################################################################### + energy_virtual_meter_hourly = dict() + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + try: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_virtual_meter_hourly " + " WHERE virtual_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_virtual_meter_hourly[virtual_meter_id] = None + else: + energy_virtual_meter_hourly[virtual_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 6.1 of equipment_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 7: for each offline meter in list, get energy input data from energy database + #################################################################################################################### + energy_offline_meter_hourly = dict() + if offline_meter_list is not None and len(offline_meter_list) > 0: + try: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_offline_meter_hourly " + " WHERE offline_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_offline_meter_hourly[offline_meter_id] = None + else: + energy_offline_meter_hourly[offline_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1] + + except Exception as e: + error_string = "Error in step 7.1 of equipment_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 8: determine common time slot to aggregate + #################################################################################################################### + + common_start_datetime_utc = start_datetime_utc + common_end_datetime_utc = end_datetime_utc + + print("Getting common time slot of energy values for all meters") + if energy_meter_hourly is not None and len(energy_meter_hourly) > 0: + for meter_id, energy_hourly in energy_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all virtual meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0: + for meter_id, energy_hourly in energy_virtual_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all offline meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0: + for meter_id, energy_hourly in energy_offline_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \ + (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \ + (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0): + # There isn't any energy data + print("There isn't any energy data") + # continue the for equipment loop to the next equipment + print("continue the for equipment loop to the next equipment") + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + return None + + print("common_start_datetime_utc: " + str(common_start_datetime_utc)) + print("common_end_datetime_utc: " + str(common_end_datetime_utc)) + + #################################################################################################################### + # Step 9: aggregate energy data in the common time slot by energy items and hourly + #################################################################################################################### + + print("Step 9: aggregate energy data in the common time slot by energy items and hourly") + aggregated_values = list() + try: + current_datetime_utc = common_start_datetime_utc + while common_start_datetime_utc is not None \ + and common_end_datetime_utc is not None \ + and current_datetime_utc <= common_end_datetime_utc: + aggregated_value = dict() + aggregated_value['start_datetime_utc'] = current_datetime_utc + aggregated_value['meta_data'] = dict() + + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + energy_item_id = meter['energy_item_id'] + actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + energy_item_id = virtual_meter['energy_item_id'] + actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if offline_meter_list is not None and len(offline_meter_list) > 0: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + energy_item_id = offline_meter['energy_item_id'] + actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + aggregated_values.append(aggregated_value) + + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + except Exception as e: + error_string = "Error in step 9 of equipment_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 10: save energy data to energy database + #################################################################################################################### + print("Step 10: save energy data to energy database") + + if len(aggregated_values) > 0: + try: + add_values = (" INSERT INTO tbl_equipment_input_item_hourly " + " (equipment_id, " + " energy_item_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for aggregated_value in aggregated_values: + for energy_item_id, actual_value in aggregated_value['meta_data'].items(): + add_values += " (" + str(equipment['id']) + "," + add_values += " " + str(energy_item_id) + "," + add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "'," + add_values += str(actual_value) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_energy_db.execute(add_values[:-2]) + cnx_energy_db.commit() + + except Exception as e: + error_string = "Error in step 10.1 of equipment_energy_input_item.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + else: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() diff --git a/myems-aggregation/equipment_energy_output_category.py b/myems-aggregation/equipment_energy_output_category.py new file mode 100644 index 00000000..8e2fadcd --- /dev/null +++ b/myems-aggregation/equipment_energy_output_category.py @@ -0,0 +1,524 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +from multiprocessing import Pool +import random +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all equipments +# Step 2: Create multiprocessing pool to call worker in parallel +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all equipments + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of equipment_energy_output_category.main " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_equipments " + " ORDER BY id ") + rows_equipments = cursor_system_db.fetchall() + + if rows_equipments is None or len(rows_equipments) == 0: + print("There isn't any equipments ") + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + + equipment_list = list() + for row in rows_equipments: + equipment_list.append({"id": row[0], "name": row[1]}) + + except Exception as e: + logger.error("Error in step 1.2 of equipment_energy_output_category.main " + str(e)) + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + print("Got all equipments in MyEMS System Database") + + # shuffle the equipment list for randomly calculating the meter hourly value + random.shuffle(equipment_list) + + ################################################################################################################ + # Step 2: Create multiprocessing pool to call worker in parallel + ################################################################################################################ + p = Pool(processes=config.pool_size) + error_list = p.map(worker, equipment_list) + p.close() + p.join() + + for error in error_list: + if error is not None and len(error) > 0: + logger.error(error) + + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of outer while + + +######################################################################################################################## +# PROCEDURES: +# Step 1: get all output meters associated with the equipment +# Step 2: get all output virtual meters associated with the equipment +# Step 3: get all output offline meters associated with the equipment +# Step 4: determine start datetime and end datetime to aggregate +# Step 5: for each meter in list, get energy output data from energy database +# Step 6: for each virtual meter in list, get energy output data from energy database +# Step 7: for each offline meter in list, get energy output data from energy database +# Step 8: determine common time slot to aggregate +# Step 9: aggregate energy data in the common time slot by energy categories and hourly +# Step 10: save energy data to energy database +# +# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter +######################################################################################################################## + +def worker(equipment): + #################################################################################################################### + # Step 1: get all output meters associated with the equipment + #################################################################################################################### + print("Step 1: get all output meters associated with the equipment " + str(equipment['name'])) + + meter_list = list() + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + error_string = "Error in step 1.1 of equipment_energy_output_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_meters m, tbl_equipments_meters em " + " WHERE m.id = em.meter_id " + " AND m.is_counted = true " + " AND em.is_output = true " + " AND em.equipment_id = %s ", + (equipment['id'],)) + rows_meters = cursor_system_db.fetchall() + + if rows_meters is not None and len(rows_meters) > 0: + for row in rows_meters: + meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 1.2 of equipment_energy_output_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 2: get all output virtual meters associated with the equipment + #################################################################################################################### + print("Step 2: get all output virtual meters associated with the equipment") + virtual_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_virtual_meters m, tbl_equipments_virtual_meters em " + " WHERE m.id = em.virtual_meter_id " + " AND m.is_counted = true " + " AND em.is_output = true " + " AND em.equipment_id = %s ", + (equipment['id'],)) + rows_virtual_meters = cursor_system_db.fetchall() + + if rows_virtual_meters is not None and len(rows_virtual_meters) > 0: + for row in rows_virtual_meters: + virtual_meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 2.1 of equipment_energy_output_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 3: get all output offline meters associated with the equipment + #################################################################################################################### + print("Step 3: get all output offline meters associated with the equipment") + + offline_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_offline_meters m, tbl_equipments_offline_meters em " + " WHERE m.id = em.offline_meter_id " + " AND m.is_counted = true " + " AND em.is_output = true " + " AND em.equipment_id = %s ", + (equipment['id'],)) + rows_offline_meters = cursor_system_db.fetchall() + + if rows_offline_meters is not None and len(rows_offline_meters) > 0: + for row in rows_offline_meters: + offline_meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 3.1 of equipment_energy_output_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + #################################################################################################################### + # stop to the next equipment if this equipment is empty + #################################################################################################################### + if (meter_list is None or len(meter_list) == 0) and \ + (virtual_meter_list is None or len(virtual_meter_list) == 0) and \ + (offline_meter_list is None or len(offline_meter_list) == 0): + print("This is an empty equipment ") + return None + + #################################################################################################################### + # Step 4: determine start datetime and end datetime to aggregate + #################################################################################################################### + print("Step 4: determine start datetime and end datetime to aggregate") + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + error_string = "Error in step 4.1 of equipment_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + try: + query = (" SELECT MAX(start_datetime_utc) " + " FROM tbl_equipment_output_category_hourly " + " WHERE equipment_id = %s ") + cursor_energy_db.execute(query, (equipment['id'],)) + row_datetime = cursor_energy_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19] + + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19]) + + except Exception as e: + error_string = "Error in step 4.2 of equipment_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 5: for each meter in list, get energy output data from energy database + #################################################################################################################### + energy_meter_hourly = dict() + try: + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_meter_hourly " + " WHERE meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_meter_hourly[meter_id] = None + else: + energy_meter_hourly[meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 5.1 of equipment_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 6: for each virtual meter in list, get energy output data from energy database + #################################################################################################################### + energy_virtual_meter_hourly = dict() + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + try: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_virtual_meter_hourly " + " WHERE virtual_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_virtual_meter_hourly[virtual_meter_id] = None + else: + energy_virtual_meter_hourly[virtual_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 6.1 of equipment_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 7: for each offline meter in list, get energy output data from energy database + #################################################################################################################### + energy_offline_meter_hourly = dict() + if offline_meter_list is not None and len(offline_meter_list) > 0: + try: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_offline_meter_hourly " + " WHERE offline_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_offline_meter_hourly[offline_meter_id] = None + else: + energy_offline_meter_hourly[offline_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1] + + except Exception as e: + error_string = "Error in step 7.1 of equipment_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 8: determine common time slot to aggregate + #################################################################################################################### + + common_start_datetime_utc = start_datetime_utc + common_end_datetime_utc = end_datetime_utc + + print("Getting common time slot of energy values for all meters") + if energy_meter_hourly is not None and len(energy_meter_hourly) > 0: + for meter_id, energy_hourly in energy_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all virtual meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0: + for meter_id, energy_hourly in energy_virtual_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all offline meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0: + for meter_id, energy_hourly in energy_offline_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \ + (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \ + (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0): + # There isn't any energy data + print("There isn't any energy data") + # continue the for equipment loop to the next equipment + print("continue the for equipment loop to the next equipment") + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + return None + + print("common_start_datetime_utc: " + str(common_start_datetime_utc)) + print("common_end_datetime_utc: " + str(common_end_datetime_utc)) + + #################################################################################################################### + # Step 9: aggregate energy data in the common time slot by energy categories and hourly + #################################################################################################################### + + print("Step 9: aggregate energy data in the common time slot by energy categories and hourly") + aggregated_values = list() + try: + current_datetime_utc = common_start_datetime_utc + while common_start_datetime_utc is not None \ + and common_end_datetime_utc is not None \ + and current_datetime_utc <= common_end_datetime_utc: + aggregated_value = dict() + aggregated_value['start_datetime_utc'] = current_datetime_utc + aggregated_value['meta_data'] = dict() + + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + energy_category_id = meter['energy_category_id'] + actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + energy_category_id = virtual_meter['energy_category_id'] + actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if offline_meter_list is not None and len(offline_meter_list) > 0: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + energy_category_id = offline_meter['energy_category_id'] + actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + aggregated_values.append(aggregated_value) + + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + except Exception as e: + error_string = "Error in step 9 of equipment_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 10: save energy data to energy database + #################################################################################################################### + print("Step 10: save energy data to energy database") + + if len(aggregated_values) > 0: + try: + add_values = (" INSERT INTO tbl_equipment_output_category_hourly " + " (equipment_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for aggregated_value in aggregated_values: + for energy_category_id, actual_value in aggregated_value['meta_data'].items(): + add_values += " (" + str(equipment['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "'," + add_values += str(actual_value) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_energy_db.execute(add_values[:-2]) + cnx_energy_db.commit() + + except Exception as e: + error_string = "Error in step 10.1 of equipment_energy_output_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + else: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() diff --git a/myems-aggregation/main.py b/myems-aggregation/main.py new file mode 100644 index 00000000..bd31f7c9 --- /dev/null +++ b/myems-aggregation/main.py @@ -0,0 +1,147 @@ +import logging +from logging.handlers import RotatingFileHandler +from multiprocessing import Process + + +import combined_equipment_energy_input_category +import combined_equipment_energy_input_item +import combined_equipment_energy_output_category + +import combined_equipment_billing_input_category +import combined_equipment_billing_input_item +import combined_equipment_billing_output_category + +import equipment_energy_input_category +import equipment_energy_input_item +import equipment_energy_output_category + +import equipment_billing_input_category +import equipment_billing_input_item +import equipment_billing_output_category + +import meter_billing + +import shopfloor_billing_input_category +import shopfloor_billing_input_item + +import shopfloor_energy_input_category +import shopfloor_energy_input_item + +import space_billing_input_category +import space_billing_input_item +import space_billing_output_category + +import space_energy_input_category +import space_energy_input_item +import space_energy_output_category + +import store_billing_input_category +import store_billing_input_item + +import store_energy_input_category +import store_energy_input_item + +import tenant_billing_input_category +import tenant_billing_input_item + +import tenant_energy_input_category +import tenant_energy_input_item + + +def main(): + """main""" + # create logger + logger = logging.getLogger('myems-aggregation') + # specifies the lowest-severity log message a logger will handle, + # where debug is the lowest built-in severity level and critical is the highest built-in severity. + # For example, if the severity level is INFO, the logger will handle only INFO, WARNING, ERROR, and CRITICAL + # messages and will ignore DEBUG messages. + logger.setLevel(logging.ERROR) + # create file handler which logs messages + fh = RotatingFileHandler('myems-aggregation.log', maxBytes=1024*1024, backupCount=1) + # create formatter and add it to the handlers + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + fh.setFormatter(formatter) + # add the handlers to logger + logger.addHandler(fh) + + # combined equipment energy input by energy categories + Process(target=combined_equipment_energy_input_category.main, args=(logger,)).start() + # combined equipment energy input by energy items + Process(target=combined_equipment_energy_input_item.main, args=(logger,)).start() + # combined equipment energy output by energy categories + Process(target=combined_equipment_energy_output_category.main, args=(logger,)).start() + + # combined equipment billing input by energy categories + Process(target=combined_equipment_billing_input_category.main, args=(logger,)).start() + # combined equipment billing input by energy items + Process(target=combined_equipment_billing_input_item.main, args=(logger,)).start() + # combined equipment billing output by energy categories + Process(target=combined_equipment_billing_output_category.main, args=(logger,)).start() + + # equipment billing input by energy categories + Process(target=equipment_billing_input_category.main, args=(logger,)).start() + # equipment billing input by energy items + Process(target=equipment_billing_input_item.main, args=(logger,)).start() + # equipment billing output by energy categories + Process(target=equipment_billing_output_category.main, args=(logger,)).start() + + # equipment energy input by energy categories + Process(target=equipment_energy_input_category.main, args=(logger,)).start() + # equipment energy input by energy items + Process(target=equipment_energy_input_item.main, args=(logger,)).start() + # equipment energy output by energy categories + Process(target=equipment_energy_output_category.main, args=(logger,)).start() + + # meter billing + Process(target=meter_billing.main, args=(logger,)).start() + + # shopfloor billing input by energy categories + Process(target=shopfloor_billing_input_category.main, args=(logger,)).start() + # shopfloor billing input by energy items + Process(target=shopfloor_billing_input_item.main, args=(logger,)).start() + + # shopfloor energy input by energy categories + Process(target=shopfloor_energy_input_category.main, args=(logger,)).start() + # shopfloor energy input by energy items + Process(target=shopfloor_energy_input_item.main, args=(logger,)).start() + + # space billing input by energy categories + Process(target=space_billing_input_category.main, args=(logger,)).start() + # space billing input by energy items + Process(target=space_billing_input_item.main, args=(logger,)).start() + # space billing output by energy categories + Process(target=space_billing_output_category.main, args=(logger,)).start() + + # space energy input by energy categories + Process(target=space_energy_input_category.main, args=(logger,)).start() + # space energy input by energy items + Process(target=space_energy_input_item.main, args=(logger,)).start() + # space energy output by energy categories + Process(target=space_energy_output_category.main, args=(logger,)).start() + + # store billing input by energy categories + Process(target=store_billing_input_category.main, args=(logger,)).start() + # store billing input by energy items + Process(target=store_billing_input_item.main, args=(logger,)).start() + + # store energy input by energy categories + Process(target=store_energy_input_category.main, args=(logger,)).start() + # store energy input by energy items + Process(target=store_energy_input_item.main, args=(logger,)).start() + + # tenant billing input by energy categories + Process(target=tenant_billing_input_category.main, args=(logger,)).start() + # tenant billing input by energy items + Process(target=tenant_billing_input_item.main, args=(logger,)).start() + + # tenant energy input by energy categories + Process(target=tenant_energy_input_category.main, args=(logger,)).start() + # tenant energy input by energy items + Process(target=tenant_energy_input_item.main, args=(logger,)).start() + + +if __name__ == '__main__': + main() + + diff --git a/myems-aggregation/meter_billing.py b/myems-aggregation/meter_billing.py new file mode 100644 index 00000000..d043fddd --- /dev/null +++ b/myems-aggregation/meter_billing.py @@ -0,0 +1,266 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +import tariff +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all meters +# for each meter in list: +# Step 2: get the latest start_datetime_utc +# Step 3: get all energy data since the latest start_datetime_utc +# Step 4: get tariffs +# Step 5: calculate billing by multiplying energy with tariff +# Step 6: save billing data to database +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all meters + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of meter_billing " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name, energy_category_id, cost_center_id " + " FROM tbl_meters " + " ORDER BY id ") + rows_meters = cursor_system_db.fetchall() + + if rows_meters is None or len(rows_meters) == 0: + print("Step 1.2: There isn't any equipments. ") + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + meter_list = list() + for row in rows_meters: + meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2], + "cost_center_id": row[3]}) + + except Exception as e: + logger.error("Error in step 1.2 of meter_billing " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Step 1.2: Got all meters from MyEMS System Database") + + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + logger.error("Error in step 1.3 of meter_billing " + str(e)) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Energy Database") + + cnx_billing_db = None + cursor_billing_db = None + try: + cnx_billing_db = mysql.connector.connect(**config.myems_billing_db) + cursor_billing_db = cnx_billing_db.cursor() + except Exception as e: + logger.error("Error in step 1.4 of meter_billing " + str(e)) + if cursor_billing_db: + cursor_billing_db.close() + if cnx_billing_db: + cnx_billing_db.close() + + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Billing Database") + + for meter in meter_list: + + ############################################################################################################ + # Step 2: get the latest start_datetime_utc + ############################################################################################################ + print("Step 2: get the latest start_datetime_utc from billing database for " + meter['name']) + try: + cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) " + " FROM tbl_meter_hourly " + " WHERE meter_id = %s ", + (meter['id'], )) + row_datetime = cursor_billing_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]) + except Exception as e: + logger.error("Error in step 2 of meter_billing " + str(e)) + # break the for meter loop + break + + ############################################################################################################ + # Step 3: get all energy data since the latest start_datetime_utc + ############################################################################################################ + print("Step 3: get all energy data since the latest start_datetime_utc") + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_meter_hourly " + " WHERE meter_id = %s AND start_datetime_utc >= %s " + " ORDER BY id ") + cursor_energy_db.execute(query, (meter['id'], start_datetime_utc, )) + rows_hourly = cursor_energy_db.fetchall() + + if rows_hourly is None or len(rows_hourly) == 0: + print("Step 3: There isn't any energy input data to calculate. ") + # continue the for meter loop + continue + + energy_dict = dict() + energy_category_list = list() + energy_category_list.append(meter['energy_category_id']) + end_datetime_utc = start_datetime_utc + for row_hourly in rows_hourly: + current_datetime_utc = row_hourly[0] + actual_value = row_hourly[1] + if energy_dict.get(current_datetime_utc) is None: + energy_dict[current_datetime_utc] = dict() + energy_dict[current_datetime_utc][meter['energy_category_id']] = actual_value + if current_datetime_utc > end_datetime_utc: + end_datetime_utc = current_datetime_utc + + ############################################################################################################ + # Step 4: get tariffs + ############################################################################################################ + print("Step 4: get tariffs") + tariff_dict = dict() + for energy_category_id in energy_category_list: + tariff_dict[energy_category_id] = tariff.get_energy_category_tariffs(meter['cost_center_id'], + energy_category_id, + start_datetime_utc, + end_datetime_utc) + ############################################################################################################ + # Step 5: calculate billing by multiplying energy with tariff + ############################################################################################################ + print("Step 5: calculate billing by multiplying energy with tariff") + billing_dict = dict() + + if len(energy_dict) > 0: + for current_datetime_utc in energy_dict.keys(): + billing_dict[current_datetime_utc] = dict() + for energy_category_id in energy_category_list: + current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc) + current_energy = energy_dict[current_datetime_utc].get(energy_category_id) + if current_tariff is not None \ + and isinstance(current_tariff, Decimal) \ + and current_energy is not None \ + and isinstance(current_energy, Decimal): + billing_dict[current_datetime_utc][energy_category_id] = \ + current_energy * current_tariff + + if len(billing_dict[current_datetime_utc]) == 0: + del billing_dict[current_datetime_utc] + + ############################################################################################################ + # Step 6: save billing data to billing database + ############################################################################################################ + print("Step 6: save billing data to billing database") + + if len(billing_dict) > 0: + try: + add_values = (" INSERT INTO tbl_meter_hourly " + " (meter_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for current_datetime_utc in billing_dict: + for energy_category_id in energy_category_list: + current_billing = billing_dict[current_datetime_utc].get(energy_category_id) + if current_billing is not None and isinstance(current_billing, Decimal): + add_values += " (" + str(meter['id']) + "," + add_values += "'" + current_datetime_utc.isoformat()[0:19] + "'," + add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_billing_db.execute(add_values[:-2]) + cnx_billing_db.commit() + except Exception as e: + logger.error("Error in step 6 of meter_billing " + str(e)) + # break the for meter loop + break + + # end of for meter loop + if cnx_system_db: + cnx_system_db.close() + if cursor_system_db: + cursor_system_db.close() + + if cnx_energy_db: + cnx_energy_db.close() + if cursor_energy_db: + cursor_energy_db.close() + + if cnx_billing_db: + cnx_billing_db.close() + if cursor_billing_db: + cursor_billing_db.close() + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of the outermost while loop diff --git a/myems-aggregation/myems-aggregation.service b/myems-aggregation/myems-aggregation.service new file mode 100644 index 00000000..deebcbbc --- /dev/null +++ b/myems-aggregation/myems-aggregation.service @@ -0,0 +1,15 @@ +[Unit] +Description=myems-aggregation daemon +After=network.target + +[Service] +User=root +Group=root +ExecStart=/usr/bin/python3 /myems-aggregation/main.py +ExecReload=/bin/kill -s HUP $MAINPID +ExecStop=/bin/kill -s TERM $MAINPID +PrivateTmp=true +Restart=always + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/myems-aggregation/shopfloor_billing_input_category.py b/myems-aggregation/shopfloor_billing_input_category.py new file mode 100644 index 00000000..4013a54a --- /dev/null +++ b/myems-aggregation/shopfloor_billing_input_category.py @@ -0,0 +1,269 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +import tariff +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all shopfloors +# for each shopfloor in list: +# Step 2: get the latest start_datetime_utc +# Step 3: get all energy input data since the latest start_datetime_utc +# Step 4: get tariffs +# Step 5: calculate billing by multiplying energy with tariff +# Step 6: save billing data to database +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all shopfloors + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of shopfloor_billing_input_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name, cost_center_id " + " FROM tbl_shopfloors " + " ORDER BY id ") + rows_shopfloors = cursor_system_db.fetchall() + + if rows_shopfloors is None or len(rows_shopfloors) == 0: + print("Step 1.2: There isn't any shopfloors. ") + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + shopfloor_list = list() + for row in rows_shopfloors: + shopfloor_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]}) + + except Exception as e: + logger.error("Error in step 1.2 of shopfloor_billing_input_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Step 1.2: Got all shopfloors from MyEMS System Database") + + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + logger.error("Error in step 1.3 of shopfloor_billing_input_category " + str(e)) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Energy Database") + + cnx_billing_db = None + cursor_billing_db = None + try: + cnx_billing_db = mysql.connector.connect(**config.myems_billing_db) + cursor_billing_db = cnx_billing_db.cursor() + except Exception as e: + logger.error("Error in step 1.4 of shopfloor_billing_input_category " + str(e)) + if cursor_billing_db: + cursor_billing_db.close() + if cnx_billing_db: + cnx_billing_db.close() + + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Billing Database") + + for shopfloor in shopfloor_list: + + ############################################################################################################ + # Step 2: get the latest start_datetime_utc + ############################################################################################################ + print("Step 2: get the latest start_datetime_utc from billing database for " + shopfloor['name']) + try: + cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) " + " FROM tbl_shopfloor_input_category_hourly " + " WHERE shopfloor_id = %s ", + (shopfloor['id'], )) + row_datetime = cursor_billing_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]) + except Exception as e: + logger.error("Error in step 2 of shopfloor_billing_input_category " + str(e)) + # break the for shopfloor loop + break + + ############################################################################################################ + # Step 3: get all energy input data since the latest start_datetime_utc + ############################################################################################################ + print("Step 3: get all energy input data since the latest start_datetime_utc") + + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_shopfloor_input_category_hourly " + " WHERE shopfloor_id = %s AND start_datetime_utc >= %s " + " ORDER BY id ") + cursor_energy_db.execute(query, (shopfloor['id'], start_datetime_utc, )) + rows_hourly = cursor_energy_db.fetchall() + + if rows_hourly is None or len(rows_hourly) == 0: + print("Step 3: There isn't any energy input data to calculate. ") + # continue the for shopfloor loop + continue + + energy_dict = dict() + energy_category_list = list() + end_datetime_utc = start_datetime_utc + for row_hourly in rows_hourly: + current_datetime_utc = row_hourly[0] + energy_category_id = row_hourly[1] + + if energy_category_id not in energy_category_list: + energy_category_list.append(energy_category_id) + + actual_value = row_hourly[2] + if energy_dict.get(current_datetime_utc) is None: + energy_dict[current_datetime_utc] = dict() + energy_dict[current_datetime_utc][energy_category_id] = actual_value + if current_datetime_utc > end_datetime_utc: + end_datetime_utc = current_datetime_utc + + ############################################################################################################ + # Step 4: get tariffs + ############################################################################################################ + print("Step 4: get tariffs") + tariff_dict = dict() + for energy_category_id in energy_category_list: + tariff_dict[energy_category_id] = tariff.get_energy_category_tariffs(shopfloor['cost_center_id'], + energy_category_id, + start_datetime_utc, + end_datetime_utc) + ############################################################################################################ + # Step 5: calculate billing by multiplying energy with tariff + ############################################################################################################ + print("Step 5: calculate billing by multiplying energy with tariff") + billing_dict = dict() + + if len(energy_dict) > 0: + for current_datetime_utc in energy_dict.keys(): + billing_dict[current_datetime_utc] = dict() + for energy_category_id in energy_category_list: + current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc) + current_energy = energy_dict[current_datetime_utc].get(energy_category_id) + if current_tariff is not None \ + and isinstance(current_tariff, Decimal) \ + and current_energy is not None \ + and isinstance(current_energy, Decimal): + billing_dict[current_datetime_utc][energy_category_id] = \ + current_energy * current_tariff + + if len(billing_dict[current_datetime_utc]) == 0: + del billing_dict[current_datetime_utc] + + ############################################################################################################ + # Step 6: save billing data to billing database + ############################################################################################################ + print("Step 6: save billing data to billing database") + + if len(billing_dict) > 0: + try: + add_values = (" INSERT INTO tbl_shopfloor_input_category_hourly " + " (shopfloor_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for current_datetime_utc in billing_dict: + for energy_category_id in energy_category_list: + current_billing = billing_dict[current_datetime_utc].get(energy_category_id) + if current_billing is not None and isinstance(current_billing, Decimal): + add_values += " (" + str(shopfloor['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + current_datetime_utc.isoformat()[0:19] + "'," + add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_billing_db.execute(add_values[:-2]) + cnx_billing_db.commit() + except Exception as e: + logger.error("Error in step 6 of shopfloor_billing_input_category " + str(e)) + # break the for shopfloor loop + break + + # end of for shopfloor loop + if cnx_system_db: + cnx_system_db.close() + if cursor_system_db: + cursor_system_db.close() + + if cnx_energy_db: + cnx_energy_db.close() + if cursor_energy_db: + cursor_energy_db.close() + + if cnx_billing_db: + cnx_billing_db.close() + if cursor_billing_db: + cursor_billing_db.close() + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of the outermost while loop diff --git a/myems-aggregation/shopfloor_billing_input_item.py b/myems-aggregation/shopfloor_billing_input_item.py new file mode 100644 index 00000000..cc47fe06 --- /dev/null +++ b/myems-aggregation/shopfloor_billing_input_item.py @@ -0,0 +1,269 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +import tariff +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all shopfloors +# for each shopfloor in list: +# Step 2: get the latest start_datetime_utc +# Step 3: get all energy input data since the latest start_datetime_utc +# Step 4: get tariffs +# Step 5: calculate billing by multiplying energy with tariff +# Step 6: save billing data to database +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all shopfloors + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of shopfloor_billing_input_item " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name, cost_center_id " + " FROM tbl_shopfloors " + " ORDER BY id ") + rows_shopfloors = cursor_system_db.fetchall() + + if rows_shopfloors is None or len(rows_shopfloors) == 0: + print("Step 1.2: There isn't any shopfloors. ") + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + shopfloor_list = list() + for row in rows_shopfloors: + shopfloor_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]}) + + except Exception as e: + logger.error("Error in step 1.2 of shopfloor_billing_input_item " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Step 1.2: Got all shopfloors from MyEMS System Database") + + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + logger.error("Error in step 1.3 of shopfloor_billing_input_item " + str(e)) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Energy Database") + + cnx_billing_db = None + cursor_billing_db = None + try: + cnx_billing_db = mysql.connector.connect(**config.myems_billing_db) + cursor_billing_db = cnx_billing_db.cursor() + except Exception as e: + logger.error("Error in step 1.4 of shopfloor_billing_input_item " + str(e)) + if cursor_billing_db: + cursor_billing_db.close() + if cnx_billing_db: + cnx_billing_db.close() + + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Billing Database") + + for shopfloor in shopfloor_list: + + ############################################################################################################ + # Step 2: get the latest start_datetime_utc + ############################################################################################################ + print("Step 2: get the latest start_datetime_utc from billing database for " + shopfloor['name']) + try: + cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) " + " FROM tbl_shopfloor_input_item_hourly " + " WHERE shopfloor_id = %s ", + (shopfloor['id'], )) + row_datetime = cursor_billing_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]) + except Exception as e: + logger.error("Error in step 2 of shopfloor_billing_input_item " + str(e)) + # break the for shopfloor loop + break + + ############################################################################################################ + # Step 3: get all energy input data since the latest start_datetime_utc + ############################################################################################################ + print("Step 3: get all energy input data since the latest start_datetime_utc") + + query = (" SELECT start_datetime_utc, energy_item_id, actual_value " + " FROM tbl_shopfloor_input_item_hourly " + " WHERE shopfloor_id = %s AND start_datetime_utc >= %s " + " ORDER BY id ") + cursor_energy_db.execute(query, (shopfloor['id'], start_datetime_utc, )) + rows_hourly = cursor_energy_db.fetchall() + + if rows_hourly is None or len(rows_hourly) == 0: + print("Step 3: There isn't any energy input data to calculate. ") + # continue the for shopfloor loop + continue + + energy_dict = dict() + energy_item_list = list() + end_datetime_utc = start_datetime_utc + for row_hourly in rows_hourly: + current_datetime_utc = row_hourly[0] + energy_item_id = row_hourly[1] + + if energy_item_id not in energy_item_list: + energy_item_list.append(energy_item_id) + + actual_value = row_hourly[2] + if energy_dict.get(current_datetime_utc) is None: + energy_dict[current_datetime_utc] = dict() + energy_dict[current_datetime_utc][energy_item_id] = actual_value + if current_datetime_utc > end_datetime_utc: + end_datetime_utc = current_datetime_utc + + ############################################################################################################ + # Step 4: get tariffs + ############################################################################################################ + print("Step 4: get tariffs") + tariff_dict = dict() + for energy_item_id in energy_item_list: + tariff_dict[energy_item_id] = tariff.get_energy_item_tariffs(shopfloor['cost_center_id'], + energy_item_id, + start_datetime_utc, + end_datetime_utc) + ############################################################################################################ + # Step 5: calculate billing by multiplying energy with tariff + ############################################################################################################ + print("Step 5: calculate billing by multiplying energy with tariff") + billing_dict = dict() + + if len(energy_dict) > 0: + for current_datetime_utc in energy_dict.keys(): + billing_dict[current_datetime_utc] = dict() + for energy_item_id in energy_item_list: + current_tariff = tariff_dict[energy_item_id].get(current_datetime_utc) + current_energy = energy_dict[current_datetime_utc].get(energy_item_id) + if current_tariff is not None \ + and isinstance(current_tariff, Decimal) \ + and current_energy is not None \ + and isinstance(current_energy, Decimal): + billing_dict[current_datetime_utc][energy_item_id] = \ + current_energy * current_tariff + + if len(billing_dict[current_datetime_utc]) == 0: + del billing_dict[current_datetime_utc] + + ############################################################################################################ + # Step 6: save billing data to billing database + ############################################################################################################ + print("Step 6: save billing data to billing database") + + if len(billing_dict) > 0: + try: + add_values = (" INSERT INTO tbl_shopfloor_input_item_hourly " + " (shopfloor_id, " + " energy_item_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for current_datetime_utc in billing_dict: + for energy_item_id in energy_item_list: + current_billing = billing_dict[current_datetime_utc].get(energy_item_id) + if current_billing is not None and isinstance(current_billing, Decimal): + add_values += " (" + str(shopfloor['id']) + "," + add_values += " " + str(energy_item_id) + "," + add_values += "'" + current_datetime_utc.isoformat()[0:19] + "'," + add_values += str(billing_dict[current_datetime_utc][energy_item_id]) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_billing_db.execute(add_values[:-2]) + cnx_billing_db.commit() + except Exception as e: + logger.error("Error in step 6 of shopfloor_billing_input_item " + str(e)) + # break the for shopfloor loop + break + + # end of for shopfloor loop + if cnx_system_db: + cnx_system_db.close() + if cursor_system_db: + cursor_system_db.close() + + if cnx_energy_db: + cnx_energy_db.close() + if cursor_energy_db: + cursor_energy_db.close() + + if cnx_billing_db: + cnx_billing_db.close() + if cursor_billing_db: + cursor_billing_db.close() + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of the outermost while loop diff --git a/myems-aggregation/shopfloor_energy_input_category.py b/myems-aggregation/shopfloor_energy_input_category.py new file mode 100644 index 00000000..6a15a265 --- /dev/null +++ b/myems-aggregation/shopfloor_energy_input_category.py @@ -0,0 +1,615 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +from multiprocessing import Pool +import random +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all shopfloors +# Step 2: Create multiprocessing pool to call worker in parallel +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all shopfloors + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of shopfloor_energy_input_category.main " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_shopfloors " + " ORDER BY id ") + rows_shopfloors = cursor_system_db.fetchall() + + if rows_shopfloors is None or len(rows_shopfloors) == 0: + print("There isn't any shopfloors ") + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + + shopfloor_list = list() + for row in rows_shopfloors: + shopfloor_list.append({"id": row[0], "name": row[1]}) + + except Exception as e: + logger.error("Error in step 1.2 of shopfloor_energy_input_category.main " + str(e)) + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + print("Got all shopfloors in MyEMS System Database") + + # shuffle the shopfloor list for randomly calculating the meter hourly value + random.shuffle(shopfloor_list) + + ################################################################################################################ + # Step 2: Create multiprocessing pool to call worker in parallel + ################################################################################################################ + p = Pool(processes=config.pool_size) + error_list = p.map(worker, shopfloor_list) + p.close() + p.join() + + for error in error_list: + if error is not None and len(error) > 0: + logger.error(error) + + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of outer while + + +######################################################################################################################## +# PROCEDURES: +# Step 1: get all input meters associated with the shopfloor +# Step 2: get all input virtual meters associated with the shopfloor +# Step 3: get all input offline meters associated with the shopfloor +# Step 4: get all equipments associated with the shopfloor +# Step 5: determine start datetime and end datetime to aggregate +# Step 6: for each meter in list, get energy input data from energy database +# Step 7: for each virtual meter in list, get energy input data from energy database +# Step 8: for each offline meter in list, get energy input data from energy database +# Step 9: for each equipment in list, get energy input data from energy database +# Step 10: determine common time slot to aggregate +# Step 11: aggregate energy data in the common time slot by energy categories and hourly +# Step 12: save energy data to energy database +# +# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter +######################################################################################################################## + +def worker(shopfloor): + #################################################################################################################### + # Step 1: get all input meters associated with the shopfloor + #################################################################################################################### + print("Step 1: get all input meters associated with the shopfloor " + str(shopfloor['name'])) + + meter_list = list() + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + error_string = "Error in step 1.1 of shopfloor_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_meters m, tbl_shopfloors_meters tm " + " WHERE m.id = tm.meter_id " + " AND m.is_counted = true " + " AND tm.shopfloor_id = %s ", + (shopfloor['id'],)) + rows_meters = cursor_system_db.fetchall() + + if rows_meters is not None and len(rows_meters) > 0: + for row in rows_meters: + meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 1.2 of shopfloor_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 2: get all input virtual meters associated with the shopfloor + #################################################################################################################### + print("Step 2: get all input virtual meters associated with the shopfloor") + virtual_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_virtual_meters m, tbl_shopfloors_virtual_meters tm " + " WHERE m.id = tm.virtual_meter_id " + " AND m.is_counted = true " + " AND tm.shopfloor_id = %s ", + (shopfloor['id'],)) + rows_virtual_meters = cursor_system_db.fetchall() + + if rows_virtual_meters is not None and len(rows_virtual_meters) > 0: + for row in rows_virtual_meters: + virtual_meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 2.1 of shopfloor_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 3: get all input offline meters associated with the shopfloor + #################################################################################################################### + print("Step 3: get all input offline meters associated with the shopfloor") + + offline_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_offline_meters m, tbl_shopfloors_offline_meters tm " + " WHERE m.id = tm.offline_meter_id " + " AND m.is_counted = true " + " AND tm.shopfloor_id = %s ", + (shopfloor['id'],)) + rows_offline_meters = cursor_system_db.fetchall() + + if rows_offline_meters is not None and len(rows_offline_meters) > 0: + for row in rows_offline_meters: + offline_meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 3.1 of shopfloor_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 4: get all equipments associated with the shopfloor + #################################################################################################################### + print("Step 4: get all equipments associated with the shopfloor") + + equipment_list = list() + + try: + cursor_system_db.execute(" SELECT e.id, e.name " + " FROM tbl_equipments e, tbl_shopfloors_equipments se " + " WHERE e.id = se.equipment_id " + " AND e.is_input_counted = true " + " AND se.shopfloor_id = %s ", + (shopfloor['id'],)) + rows_equipments = cursor_system_db.fetchall() + + if rows_equipments is not None and len(rows_equipments) > 0: + for row in rows_equipments: + equipment_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 4 of shopfloor_energy_input_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + #################################################################################################################### + # stop to the next shopfloor if this shopfloor is empty + #################################################################################################################### + if (meter_list is None or len(meter_list) == 0) and \ + (virtual_meter_list is None or len(virtual_meter_list) == 0) and \ + (offline_meter_list is None or len(offline_meter_list) == 0) and \ + (equipment_list is None or len(equipment_list) == 0): + print("This is an empty shopfloor ") + return None + + #################################################################################################################### + # Step 5: determine start datetime and end datetime to aggregate + #################################################################################################################### + print("Step 5: determine start datetime and end datetime to aggregate") + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + error_string = "Error in step 5.1 of shopfloor_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + try: + query = (" SELECT MAX(start_datetime_utc) " + " FROM tbl_shopfloor_input_category_hourly " + " WHERE shopfloor_id = %s ") + cursor_energy_db.execute(query, (shopfloor['id'],)) + row_datetime = cursor_energy_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19] + + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19]) + + except Exception as e: + error_string = "Error in step 5.2 of shopfloor_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 6: for each meter in list, get energy input data from energy database + #################################################################################################################### + energy_meter_hourly = dict() + try: + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_meter_hourly " + " WHERE meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_meter_hourly[meter_id] = None + else: + energy_meter_hourly[meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 6.1 of shopfloor_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 7: for each virtual meter in list, get energy input data from energy database + #################################################################################################################### + energy_virtual_meter_hourly = dict() + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + try: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_virtual_meter_hourly " + " WHERE virtual_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_virtual_meter_hourly[virtual_meter_id] = None + else: + energy_virtual_meter_hourly[virtual_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 7.1 of shopfloor_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 8: for each offline meter in list, get energy input data from energy database + #################################################################################################################### + energy_offline_meter_hourly = dict() + if offline_meter_list is not None and len(offline_meter_list) > 0: + try: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_offline_meter_hourly " + " WHERE offline_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_offline_meter_hourly[offline_meter_id] = None + else: + energy_offline_meter_hourly[offline_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1] + + except Exception as e: + error_string = "Error in step 8.1 of shopfloor_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 9: for each equipment in list, get energy input data from energy database + #################################################################################################################### + energy_equipment_hourly = dict() + if equipment_list is not None and len(equipment_list) > 0: + try: + for equipment in equipment_list: + equipment_id = str(equipment['id']) + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_equipment_input_category_hourly " + " WHERE equipment_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (equipment_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_equipment_hourly[equipment_id] = None + else: + energy_equipment_hourly[equipment_id] = dict() + for row_value in rows_energy_values: + current_datetime_utc = row_value[0] + if current_datetime_utc not in energy_equipment_hourly[equipment_id]: + energy_equipment_hourly[equipment_id][current_datetime_utc] = dict() + energy_category_id = row_value[1] + actual_value = row_value[2] + energy_equipment_hourly[equipment_id][current_datetime_utc][energy_category_id] = \ + actual_value + except Exception as e: + error_string = "Error in step 9 of shopfloor_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 10: determine common time slot to aggregate + #################################################################################################################### + + common_start_datetime_utc = start_datetime_utc + common_end_datetime_utc = end_datetime_utc + + print("Getting common time slot of energy values for all meters") + if energy_meter_hourly is not None and len(energy_meter_hourly) > 0: + for meter_id, energy_hourly in energy_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all virtual meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0: + for meter_id, energy_hourly in energy_virtual_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all offline meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0: + for meter_id, energy_hourly in energy_offline_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all equipments...") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_equipment_hourly is not None and len(energy_equipment_hourly) > 0: + for equipment_id, energy_hourly in energy_equipment_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \ + (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \ + (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0) and \ + (energy_equipment_hourly is None or len(energy_equipment_hourly) == 0): + # There isn't any energy data + print("There isn't any energy data") + # continue the for shopfloor loop to the next shopfloor + print("continue the for shopfloor loop to the next shopfloor") + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + return None + + print("common_start_datetime_utc: " + str(common_start_datetime_utc)) + print("common_end_datetime_utc: " + str(common_end_datetime_utc)) + + #################################################################################################################### + # Step 11: aggregate energy data in the common time slot by energy categories and hourly + #################################################################################################################### + + print("Step 11: aggregate energy data in the common time slot by energy categories and hourly") + aggregated_values = list() + try: + current_datetime_utc = common_start_datetime_utc + while common_start_datetime_utc is not None \ + and common_end_datetime_utc is not None \ + and current_datetime_utc <= common_end_datetime_utc: + aggregated_value = dict() + aggregated_value['start_datetime_utc'] = current_datetime_utc + aggregated_value['meta_data'] = dict() + + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + energy_category_id = meter['energy_category_id'] + actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + energy_category_id = virtual_meter['energy_category_id'] + actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if offline_meter_list is not None and len(offline_meter_list) > 0: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + energy_category_id = offline_meter['energy_category_id'] + actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if equipment_list is not None and len(equipment_list) > 0: + for equipment in equipment_list: + equipment_id = str(equipment['id']) + meta_data_dict = energy_equipment_hourly[equipment_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_category_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + aggregated_values.append(aggregated_value) + + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + except Exception as e: + error_string = "Error in step 11 of shopfloor_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 12: save energy data to energy database + #################################################################################################################### + print("Step 12: save energy data to energy database") + + if len(aggregated_values) > 0: + try: + add_values = (" INSERT INTO tbl_shopfloor_input_category_hourly " + " (shopfloor_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for aggregated_value in aggregated_values: + for energy_category_id, actual_value in aggregated_value['meta_data'].items(): + add_values += " (" + str(shopfloor['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "'," + add_values += str(actual_value) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_energy_db.execute(add_values[:-2]) + cnx_energy_db.commit() + + except Exception as e: + error_string = "Error in step 12.1 of shopfloor_energy_input_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + else: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() diff --git a/myems-aggregation/shopfloor_energy_input_item.py b/myems-aggregation/shopfloor_energy_input_item.py new file mode 100644 index 00000000..e144ff55 --- /dev/null +++ b/myems-aggregation/shopfloor_energy_input_item.py @@ -0,0 +1,618 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +from multiprocessing import Pool +import random +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all shopfloors +# Step 2: Create multiprocessing pool to call worker in parallel +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all shopfloors + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of shopfloor_energy_input_item.main " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_shopfloors " + " ORDER BY id ") + rows_shopfloors = cursor_system_db.fetchall() + + if rows_shopfloors is None or len(rows_shopfloors) == 0: + print("There isn't any shopfloors ") + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + + shopfloor_list = list() + for row in rows_shopfloors: + shopfloor_list.append({"id": row[0], "name": row[1]}) + + except Exception as e: + logger.error("Error in step 1.2 of shopfloor_energy_input_item.main " + str(e)) + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + print("Got all shopfloors in MyEMS System Database") + + # shuffle the shopfloor list for randomly calculating the meter hourly value + random.shuffle(shopfloor_list) + + ################################################################################################################ + # Step 2: Create multiprocessing pool to call worker in parallel + ################################################################################################################ + p = Pool(processes=config.pool_size) + error_list = p.map(worker, shopfloor_list) + p.close() + p.join() + + for error in error_list: + if error is not None and len(error) > 0: + logger.error(error) + + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of outer while + + +######################################################################################################################## +# PROCEDURES: +# Step 1: get all input meters associated with the shopfloor +# Step 2: get all input virtual meters associated with the shopfloor +# Step 3: get all input offline meters associated with the shopfloor +# Step 4: get all equipments associated with the shopfloor +# Step 5: determine start datetime and end datetime to aggregate +# Step 6: for each meter in list, get energy input data from energy database +# Step 7: for each virtual meter in list, get energy input data from energy database +# Step 8: for each offline meter in list, get energy input data from energy database +# Step 9: for each equipment in list, get energy input data from energy database +# Step 10: determine common time slot to aggregate +# Step 11: aggregate energy data in the common time slot by energy items and hourly +# Step 12: save energy data to energy database +# +# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter +######################################################################################################################## + +def worker(shopfloor): + #################################################################################################################### + # Step 1: get all input meters associated with the shopfloor + #################################################################################################################### + print("Step 1: get all input meters associated with the shopfloor " + str(shopfloor['name'])) + + meter_list = list() + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + error_string = "Error in step 1.1 of shopfloor_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_meters m, tbl_shopfloors_meters tm " + " WHERE m.id = tm.meter_id " + " AND m.is_counted = true " + " AND m.energy_item_id is NOT NULL " + " AND tm.shopfloor_id = %s ", + (shopfloor['id'],)) + rows_meters = cursor_system_db.fetchall() + + if rows_meters is not None and len(rows_meters) > 0: + for row in rows_meters: + meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 1.2 of shopfloor_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 2: get all input virtual meters associated with the shopfloor + #################################################################################################################### + print("Step 2: get all input virtual meters associated with the shopfloor") + virtual_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_virtual_meters m, tbl_shopfloors_virtual_meters tm " + " WHERE m.id = tm.virtual_meter_id " + " AND m.energy_item_id is NOT NULL " + " AND m.is_counted = true " + " AND tm.shopfloor_id = %s ", + (shopfloor['id'],)) + rows_virtual_meters = cursor_system_db.fetchall() + + if rows_virtual_meters is not None and len(rows_virtual_meters) > 0: + for row in rows_virtual_meters: + virtual_meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 2.1 of shopfloor_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 3: get all input offline meters associated with the shopfloor + #################################################################################################################### + print("Step 3: get all input offline meters associated with the shopfloor") + + offline_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_offline_meters m, tbl_shopfloors_offline_meters tm " + " WHERE m.id = tm.offline_meter_id " + " AND m.energy_item_id is NOT NULL " + " AND m.is_counted = true " + " AND tm.shopfloor_id = %s ", + (shopfloor['id'],)) + rows_offline_meters = cursor_system_db.fetchall() + + if rows_offline_meters is not None and len(rows_offline_meters) > 0: + for row in rows_offline_meters: + offline_meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 3.1 of shopfloor_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 4: get all equipments associated with the shopfloor + #################################################################################################################### + print("Step 4: get all equipments associated with the shopfloor") + + equipment_list = list() + + try: + cursor_system_db.execute(" SELECT e.id, e.name " + " FROM tbl_equipments e, tbl_shopfloors_equipments se " + " WHERE e.id = se.equipment_id " + " AND e.is_input_counted = true " + " AND se.shopfloor_id = %s ", + (shopfloor['id'],)) + rows_equipments = cursor_system_db.fetchall() + + if rows_equipments is not None and len(rows_equipments) > 0: + for row in rows_equipments: + equipment_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 4 of shopfloor_energy_input_item.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + #################################################################################################################### + # stop to the next shopfloor if this shopfloor is empty + #################################################################################################################### + if (meter_list is None or len(meter_list) == 0) and \ + (virtual_meter_list is None or len(virtual_meter_list) == 0) and \ + (offline_meter_list is None or len(offline_meter_list) == 0) and \ + (equipment_list is None or len(equipment_list) == 0): + print("This is an empty shopfloor ") + return None + + #################################################################################################################### + # Step 5: determine start datetime and end datetime to aggregate + #################################################################################################################### + print("Step 5: determine start datetime and end datetime to aggregate") + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + error_string = "Error in step 5.1 of shopfloor_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + try: + query = (" SELECT MAX(start_datetime_utc) " + " FROM tbl_shopfloor_input_item_hourly " + " WHERE shopfloor_id = %s ") + cursor_energy_db.execute(query, (shopfloor['id'],)) + row_datetime = cursor_energy_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19] + + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19]) + + except Exception as e: + error_string = "Error in step 5.2 of shopfloor_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 6: for each meter in list, get energy input data from energy database + #################################################################################################################### + energy_meter_hourly = dict() + try: + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_meter_hourly " + " WHERE meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_meter_hourly[meter_id] = None + else: + energy_meter_hourly[meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 6.1 of shopfloor_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 7: for each virtual meter in list, get energy input data from energy database + #################################################################################################################### + energy_virtual_meter_hourly = dict() + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + try: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_virtual_meter_hourly " + " WHERE virtual_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_virtual_meter_hourly[virtual_meter_id] = None + else: + energy_virtual_meter_hourly[virtual_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 7.1 of shopfloor_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 8: for each offline meter in list, get energy input data from energy database + #################################################################################################################### + energy_offline_meter_hourly = dict() + if offline_meter_list is not None and len(offline_meter_list) > 0: + try: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_offline_meter_hourly " + " WHERE offline_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_offline_meter_hourly[offline_meter_id] = None + else: + energy_offline_meter_hourly[offline_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1] + + except Exception as e: + error_string = "Error in step 8.1 of shopfloor_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 9: for each equipment in list, get energy input data from energy database + #################################################################################################################### + energy_equipment_hourly = dict() + if equipment_list is not None and len(equipment_list) > 0: + try: + for equipment in equipment_list: + equipment_id = str(equipment['id']) + query = (" SELECT start_datetime_utc, energy_item_id, actual_value " + " FROM tbl_equipment_input_item_hourly " + " WHERE equipment_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (equipment_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_equipment_hourly[equipment_id] = None + else: + energy_equipment_hourly[equipment_id] = dict() + for row_value in rows_energy_values: + current_datetime_utc = row_value[0] + if current_datetime_utc not in energy_equipment_hourly[equipment_id]: + energy_equipment_hourly[equipment_id][current_datetime_utc] = dict() + energy_item_id = row_value[1] + actual_value = row_value[2] + energy_equipment_hourly[equipment_id][current_datetime_utc][energy_item_id] = \ + actual_value + except Exception as e: + error_string = "Error in step 9 of shopfloor_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 10: determine common time slot to aggregate + #################################################################################################################### + + common_start_datetime_utc = start_datetime_utc + common_end_datetime_utc = end_datetime_utc + + print("Getting common time slot of energy values for all meters") + if energy_meter_hourly is not None and len(energy_meter_hourly) > 0: + for meter_id, energy_hourly in energy_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all virtual meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0: + for meter_id, energy_hourly in energy_virtual_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all offline meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0: + for meter_id, energy_hourly in energy_offline_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all equipments...") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_equipment_hourly is not None and len(energy_equipment_hourly) > 0: + for equipment_id, energy_hourly in energy_equipment_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \ + (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \ + (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0) and \ + (energy_equipment_hourly is None or len(energy_equipment_hourly) == 0): + # There isn't any energy data + print("There isn't any energy data") + # continue the for shopfloor loop to the next shopfloor + print("continue the for shopfloor loop to the next shopfloor") + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + return None + + print("common_start_datetime_utc: " + str(common_start_datetime_utc)) + print("common_end_datetime_utc: " + str(common_end_datetime_utc)) + + #################################################################################################################### + # Step 11: aggregate energy data in the common time slot by energy items and hourly + #################################################################################################################### + + print("Step 11: aggregate energy data in the common time slot by energy items and hourly") + aggregated_values = list() + try: + current_datetime_utc = common_start_datetime_utc + while common_start_datetime_utc is not None \ + and common_end_datetime_utc is not None \ + and current_datetime_utc <= common_end_datetime_utc: + aggregated_value = dict() + aggregated_value['start_datetime_utc'] = current_datetime_utc + aggregated_value['meta_data'] = dict() + + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + energy_item_id = meter['energy_item_id'] + actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + energy_item_id = virtual_meter['energy_item_id'] + actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if offline_meter_list is not None and len(offline_meter_list) > 0: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + energy_item_id = offline_meter['energy_item_id'] + actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if equipment_list is not None and len(equipment_list) > 0: + for equipment in equipment_list: + equipment_id = str(equipment['id']) + meta_data_dict = energy_equipment_hourly[equipment_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_item_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + aggregated_values.append(aggregated_value) + + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + except Exception as e: + error_string = "Error in step 11 of shopfloor_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 12: save energy data to energy database + #################################################################################################################### + print("Step 12: save energy data to energy database") + + if len(aggregated_values) > 0: + try: + add_values = (" INSERT INTO tbl_shopfloor_input_item_hourly " + " (shopfloor_id, " + " energy_item_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for aggregated_value in aggregated_values: + for energy_item_id, actual_value in aggregated_value['meta_data'].items(): + add_values += " (" + str(shopfloor['id']) + "," + add_values += " " + str(energy_item_id) + "," + add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "'," + add_values += str(actual_value) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_energy_db.execute(add_values[:-2]) + cnx_energy_db.commit() + + except Exception as e: + error_string = "Error in step 12.1 of shopfloor_energy_input_item.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + else: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() diff --git a/myems-aggregation/space_billing_input_category.py b/myems-aggregation/space_billing_input_category.py new file mode 100644 index 00000000..fdfbcf2e --- /dev/null +++ b/myems-aggregation/space_billing_input_category.py @@ -0,0 +1,269 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +import tariff +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all spaces +# for each space in list: +# Step 2: get the latest start_datetime_utc +# Step 3: get all energy input data since the latest start_datetime_utc +# Step 4: get tariffs +# Step 5: calculate billing by multiplying energy with tariff +# Step 6: save billing data to database +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all spaces + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of space_billing_input_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name, cost_center_id " + " FROM tbl_spaces " + " ORDER BY id ") + rows_spaces = cursor_system_db.fetchall() + + if rows_spaces is None or len(rows_spaces) == 0: + print("Step 1.2: There isn't any spaces. ") + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + space_list = list() + for row in rows_spaces: + space_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]}) + + except Exception as e: + logger.error("Error in step 1.2 of space_billing_input_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Step 1.2: Got all spaces from MyEMS System Database") + + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + logger.error("Error in step 1.3 of space_billing_input_category " + str(e)) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Energy Database") + + cnx_billing_db = None + cursor_billing_db = None + try: + cnx_billing_db = mysql.connector.connect(**config.myems_billing_db) + cursor_billing_db = cnx_billing_db.cursor() + except Exception as e: + logger.error("Error in step 1.4 of space_billing_input_category " + str(e)) + if cursor_billing_db: + cursor_billing_db.close() + if cnx_billing_db: + cnx_billing_db.close() + + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Billing Database") + + for space in space_list: + + ############################################################################################################ + # Step 2: get the latest start_datetime_utc + ############################################################################################################ + print("Step 2: get the latest start_datetime_utc from billing database for " + space['name']) + try: + cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) " + " FROM tbl_space_input_category_hourly " + " WHERE space_id = %s ", + (space['id'], )) + row_datetime = cursor_billing_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]) + except Exception as e: + logger.error("Error in step 2 of space_billing_input_category " + str(e)) + # break the for space loop + break + + ############################################################################################################ + # Step 3: get all energy input data since the latest start_datetime_utc + ############################################################################################################ + print("Step 3: get all energy input data since the latest start_datetime_utc") + + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_space_input_category_hourly " + " WHERE space_id = %s AND start_datetime_utc >= %s " + " ORDER BY id ") + cursor_energy_db.execute(query, (space['id'], start_datetime_utc, )) + rows_hourly = cursor_energy_db.fetchall() + + if rows_hourly is None or len(rows_hourly) == 0: + print("Step 3: There isn't any energy input data to calculate. ") + # continue the for space loop + continue + + energy_dict = dict() + energy_category_list = list() + end_datetime_utc = start_datetime_utc + for row_hourly in rows_hourly: + current_datetime_utc = row_hourly[0] + energy_category_id = row_hourly[1] + + if energy_category_id not in energy_category_list: + energy_category_list.append(energy_category_id) + + actual_value = row_hourly[2] + if energy_dict.get(current_datetime_utc) is None: + energy_dict[current_datetime_utc] = dict() + energy_dict[current_datetime_utc][energy_category_id] = actual_value + if current_datetime_utc > end_datetime_utc: + end_datetime_utc = current_datetime_utc + + ############################################################################################################ + # Step 4: get tariffs + ############################################################################################################ + print("Step 4: get tariffs") + tariff_dict = dict() + for energy_category_id in energy_category_list: + tariff_dict[energy_category_id] = tariff.get_energy_category_tariffs(space['cost_center_id'], + energy_category_id, + start_datetime_utc, + end_datetime_utc) + ############################################################################################################ + # Step 5: calculate billing by multiplying energy with tariff + ############################################################################################################ + print("Step 5: calculate billing by multiplying energy with tariff") + billing_dict = dict() + + if len(energy_dict) > 0: + for current_datetime_utc in energy_dict.keys(): + billing_dict[current_datetime_utc] = dict() + for energy_category_id in energy_category_list: + current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc) + current_energy = energy_dict[current_datetime_utc].get(energy_category_id) + if current_tariff is not None \ + and isinstance(current_tariff, Decimal) \ + and current_energy is not None \ + and isinstance(current_energy, Decimal): + billing_dict[current_datetime_utc][energy_category_id] = \ + current_energy * current_tariff + + if len(billing_dict[current_datetime_utc]) == 0: + del billing_dict[current_datetime_utc] + + ############################################################################################################ + # Step 6: save billing data to billing database + ############################################################################################################ + print("Step 6: save billing data to billing database") + + if len(billing_dict) > 0: + try: + add_values = (" INSERT INTO tbl_space_input_category_hourly " + " (space_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for current_datetime_utc in billing_dict: + for energy_category_id in energy_category_list: + current_billing = billing_dict[current_datetime_utc].get(energy_category_id) + if current_billing is not None and isinstance(current_billing, Decimal): + add_values += " (" + str(space['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + current_datetime_utc.isoformat()[0:19] + "'," + add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_billing_db.execute(add_values[:-2]) + cnx_billing_db.commit() + except Exception as e: + logger.error("Error in step 6 of space_billing_input_category " + str(e)) + # break the for space loop + break + + # end of for space loop + if cnx_system_db: + cnx_system_db.close() + if cursor_system_db: + cursor_system_db.close() + + if cnx_energy_db: + cnx_energy_db.close() + if cursor_energy_db: + cursor_energy_db.close() + + if cnx_billing_db: + cnx_billing_db.close() + if cursor_billing_db: + cursor_billing_db.close() + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of the outermost while loop diff --git a/myems-aggregation/space_billing_input_item.py b/myems-aggregation/space_billing_input_item.py new file mode 100644 index 00000000..40b7620c --- /dev/null +++ b/myems-aggregation/space_billing_input_item.py @@ -0,0 +1,269 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +import tariff +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all spaces +# for each space in list: +# Step 2: get the latest start_datetime_utc +# Step 3: get all energy input data since the latest start_datetime_utc +# Step 4: get tariffs +# Step 5: calculate billing by multiplying energy with tariff +# Step 6: save billing data to database +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all spaces + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of space_billing_input_item " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name, cost_center_id " + " FROM tbl_spaces " + " ORDER BY id ") + rows_spaces = cursor_system_db.fetchall() + + if rows_spaces is None or len(rows_spaces) == 0: + print("Step 1.2: There isn't any spaces. ") + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + space_list = list() + for row in rows_spaces: + space_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]}) + + except Exception as e: + logger.error("Error in step 1.2 of space_billing_input_item " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Step 1.2: Got all spaces from MyEMS System Database") + + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + logger.error("Error in step 1.3 of space_billing_input_item " + str(e)) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Energy Database") + + cnx_billing_db = None + cursor_billing_db = None + try: + cnx_billing_db = mysql.connector.connect(**config.myems_billing_db) + cursor_billing_db = cnx_billing_db.cursor() + except Exception as e: + logger.error("Error in step 1.4 of space_billing_input_item " + str(e)) + if cursor_billing_db: + cursor_billing_db.close() + if cnx_billing_db: + cnx_billing_db.close() + + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Billing Database") + + for space in space_list: + + ############################################################################################################ + # Step 2: get the latest start_datetime_utc + ############################################################################################################ + print("Step 2: get the latest start_datetime_utc from billing database for " + space['name']) + try: + cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) " + " FROM tbl_space_input_item_hourly " + " WHERE space_id = %s ", + (space['id'], )) + row_datetime = cursor_billing_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]) + except Exception as e: + logger.error("Error in step 2 of space_billing_input_item " + str(e)) + # break the for space loop + break + + ############################################################################################################ + # Step 3: get all energy input data since the latest start_datetime_utc + ############################################################################################################ + print("Step 3: get all energy input data since the latest start_datetime_utc") + + query = (" SELECT start_datetime_utc, energy_item_id, actual_value " + " FROM tbl_space_input_item_hourly " + " WHERE space_id = %s AND start_datetime_utc >= %s " + " ORDER BY id ") + cursor_energy_db.execute(query, (space['id'], start_datetime_utc, )) + rows_hourly = cursor_energy_db.fetchall() + + if rows_hourly is None or len(rows_hourly) == 0: + print("Step 3: There isn't any energy input data to calculate. ") + # continue the for space loop + continue + + energy_dict = dict() + energy_item_list = list() + end_datetime_utc = start_datetime_utc + for row_hourly in rows_hourly: + current_datetime_utc = row_hourly[0] + energy_item_id = row_hourly[1] + + if energy_item_id not in energy_item_list: + energy_item_list.append(energy_item_id) + + actual_value = row_hourly[2] + if energy_dict.get(current_datetime_utc) is None: + energy_dict[current_datetime_utc] = dict() + energy_dict[current_datetime_utc][energy_item_id] = actual_value + if current_datetime_utc > end_datetime_utc: + end_datetime_utc = current_datetime_utc + + ############################################################################################################ + # Step 4: get tariffs + ############################################################################################################ + print("Step 4: get tariffs") + tariff_dict = dict() + for energy_item_id in energy_item_list: + tariff_dict[energy_item_id] = tariff.get_energy_item_tariffs(space['cost_center_id'], + energy_item_id, + start_datetime_utc, + end_datetime_utc) + ############################################################################################################ + # Step 5: calculate billing by multiplying energy with tariff + ############################################################################################################ + print("Step 5: calculate billing by multiplying energy with tariff") + billing_dict = dict() + + if len(energy_dict) > 0: + for current_datetime_utc in energy_dict.keys(): + billing_dict[current_datetime_utc] = dict() + for energy_item_id in energy_item_list: + current_tariff = tariff_dict[energy_item_id].get(current_datetime_utc) + current_energy = energy_dict[current_datetime_utc].get(energy_item_id) + if current_tariff is not None \ + and isinstance(current_tariff, Decimal) \ + and current_energy is not None \ + and isinstance(current_energy, Decimal): + billing_dict[current_datetime_utc][energy_item_id] = \ + current_energy * current_tariff + + if len(billing_dict[current_datetime_utc]) == 0: + del billing_dict[current_datetime_utc] + + ############################################################################################################ + # Step 6: save billing data to billing database + ############################################################################################################ + print("Step 6: save billing data to billing database") + + if len(billing_dict) > 0: + try: + add_values = (" INSERT INTO tbl_space_input_item_hourly " + " (space_id, " + " energy_item_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for current_datetime_utc in billing_dict: + for energy_item_id in energy_item_list: + current_billing = billing_dict[current_datetime_utc].get(energy_item_id) + if current_billing is not None and isinstance(current_billing, Decimal): + add_values += " (" + str(space['id']) + "," + add_values += " " + str(energy_item_id) + "," + add_values += "'" + current_datetime_utc.isoformat()[0:19] + "'," + add_values += str(billing_dict[current_datetime_utc][energy_item_id]) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_billing_db.execute(add_values[:-2]) + cnx_billing_db.commit() + except Exception as e: + logger.error("Error in step 6 of space_billing_input_item " + str(e)) + # break the for space loop + break + + # end of for space loop + if cnx_system_db: + cnx_system_db.close() + if cursor_system_db: + cursor_system_db.close() + + if cnx_energy_db: + cnx_energy_db.close() + if cursor_energy_db: + cursor_energy_db.close() + + if cnx_billing_db: + cnx_billing_db.close() + if cursor_billing_db: + cursor_billing_db.close() + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of the outermost while loop diff --git a/myems-aggregation/space_billing_output_category.py b/myems-aggregation/space_billing_output_category.py new file mode 100644 index 00000000..71ba2e8d --- /dev/null +++ b/myems-aggregation/space_billing_output_category.py @@ -0,0 +1,269 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +import tariff +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all spaces +# for each space in list: +# Step 2: get the latest start_datetime_utc +# Step 3: get all energy input data since the latest start_datetime_utc +# Step 4: get tariffs +# Step 5: calculate billing by multiplying energy with tariff +# Step 6: save billing data to database +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all spaces + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of space_billing_output_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name, cost_center_id " + " FROM tbl_spaces " + " ORDER BY id ") + rows_spaces = cursor_system_db.fetchall() + + if rows_spaces is None or len(rows_spaces) == 0: + print("Step 1.2: There isn't any spaces. ") + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + space_list = list() + for row in rows_spaces: + space_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]}) + + except Exception as e: + logger.error("Error in step 1.2 of space_billing_output_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Step 1.2: Got all spaces from MyEMS System Database") + + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + logger.error("Error in step 1.3 of space_billing_output_category " + str(e)) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Energy Database") + + cnx_billing_db = None + cursor_billing_db = None + try: + cnx_billing_db = mysql.connector.connect(**config.myems_billing_db) + cursor_billing_db = cnx_billing_db.cursor() + except Exception as e: + logger.error("Error in step 1.4 of space_billing_output_category " + str(e)) + if cursor_billing_db: + cursor_billing_db.close() + if cnx_billing_db: + cnx_billing_db.close() + + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Billing Database") + + for space in space_list: + + ############################################################################################################ + # Step 2: get the latest start_datetime_utc + ############################################################################################################ + print("Step 2: get the latest start_datetime_utc from billing database for " + space['name']) + try: + cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) " + " FROM tbl_space_output_category_hourly " + " WHERE space_id = %s ", + (space['id'], )) + row_datetime = cursor_billing_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]) + except Exception as e: + logger.error("Error in step 2 of space_billing_output_category " + str(e)) + # break the for space loop + break + + ############################################################################################################ + # Step 3: get all energy output data since the latest start_datetime_utc + ############################################################################################################ + print("Step 3: get all energy output data since the latest start_datetime_utc") + + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_space_output_category_hourly " + " WHERE space_id = %s AND start_datetime_utc >= %s " + " ORDER BY id ") + cursor_energy_db.execute(query, (space['id'], start_datetime_utc, )) + rows_hourly = cursor_energy_db.fetchall() + + if rows_hourly is None or len(rows_hourly) == 0: + print("Step 3: There isn't any energy output data to calculate. ") + # continue the for space loop + continue + + energy_dict = dict() + energy_category_list = list() + end_datetime_utc = start_datetime_utc + for row_hourly in rows_hourly: + current_datetime_utc = row_hourly[0] + energy_category_id = row_hourly[1] + + if energy_category_id not in energy_category_list: + energy_category_list.append(energy_category_id) + + actual_value = row_hourly[2] + if energy_dict.get(current_datetime_utc) is None: + energy_dict[current_datetime_utc] = dict() + energy_dict[current_datetime_utc][energy_category_id] = actual_value + if current_datetime_utc > end_datetime_utc: + end_datetime_utc = current_datetime_utc + + ############################################################################################################ + # Step 4: get tariffs + ############################################################################################################ + print("Step 4: get tariffs") + tariff_dict = dict() + for energy_category_id in energy_category_list: + tariff_dict[energy_category_id] = tariff.get_energy_category_tariffs(space['cost_center_id'], + energy_category_id, + start_datetime_utc, + end_datetime_utc) + ############################################################################################################ + # Step 5: calculate billing by multiplying energy with tariff + ############################################################################################################ + print("Step 5: calculate billing by multiplying energy with tariff") + billing_dict = dict() + + if len(energy_dict) > 0: + for current_datetime_utc in energy_dict.keys(): + billing_dict[current_datetime_utc] = dict() + for energy_category_id in energy_category_list: + current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc) + current_energy = energy_dict[current_datetime_utc].get(energy_category_id) + if current_tariff is not None \ + and isinstance(current_tariff, Decimal) \ + and current_energy is not None \ + and isinstance(current_energy, Decimal): + billing_dict[current_datetime_utc][energy_category_id] = \ + current_energy * current_tariff + + if len(billing_dict[current_datetime_utc]) == 0: + del billing_dict[current_datetime_utc] + + ############################################################################################################ + # Step 6: save billing data to billing database + ############################################################################################################ + print("Step 6: save billing data to billing database") + + if len(billing_dict) > 0: + try: + add_values = (" INSERT INTO tbl_space_output_category_hourly " + " (space_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for current_datetime_utc in billing_dict: + for energy_category_id in energy_category_list: + current_billing = billing_dict[current_datetime_utc].get(energy_category_id) + if current_billing is not None and isinstance(current_billing, Decimal): + add_values += " (" + str(space['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + current_datetime_utc.isoformat()[0:19] + "'," + add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_billing_db.execute(add_values[:-2]) + cnx_billing_db.commit() + except Exception as e: + logger.error("Error in step 6 of space_billing_output_category " + str(e)) + # break the for space loop + break + + # end of for space loop + if cnx_system_db: + cnx_system_db.close() + if cursor_system_db: + cursor_system_db.close() + + if cnx_energy_db: + cnx_energy_db.close() + if cursor_energy_db: + cursor_energy_db.close() + + if cnx_billing_db: + cnx_billing_db.close() + if cursor_billing_db: + cursor_billing_db.close() + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of the outermost while loop diff --git a/myems-aggregation/space_energy_input_category.py b/myems-aggregation/space_energy_input_category.py new file mode 100644 index 00000000..ea05bcbe --- /dev/null +++ b/myems-aggregation/space_energy_input_category.py @@ -0,0 +1,1081 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +from multiprocessing import Pool +import random +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all spaces +# Step 2: Create multiprocessing pool to call worker in parallel +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all spaces + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of space_energy_input_category.main " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_spaces " + " ORDER BY id ") + rows_spaces = cursor_system_db.fetchall() + + if rows_spaces is None or len(rows_spaces) == 0: + print("There isn't any spaces ") + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + + space_list = list() + for row in rows_spaces: + space_list.append({"id": row[0], "name": row[1]}) + + except Exception as e: + logger.error("Error in step 1.2 of space_energy_input_category.main " + str(e)) + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + print("Got all spaces in MyEMS System Database") + + # shuffle the space list for randomly calculating the meter hourly value + random.shuffle(space_list) + + ################################################################################################################ + # Step 2: Create multiprocessing pool to call worker in parallel + ################################################################################################################ + p = Pool(processes=config.pool_size) + error_list = p.map(worker, space_list) + p.close() + p.join() + + for error in error_list: + if error is not None and len(error) > 0: + logger.error(error) + + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of outer while + + +######################################################################################################################## +# PROCEDURES: +# Step 1: get all input meters associated with the space +# Step 2: get all input virtual meters associated with the space +# Step 3: get all input offline meters associated with the space +# Step 4: get all combined equipments associated with the space +# Step 5: get all equipments associated with the space +# Step 6: get all shopfloors associated with the space +# Step 7: get all stores associated with the space +# Step 8: get all tenants associated with the space +# Step 9: get all child spaces associated with the space +# Step 10: determine start datetime and end datetime to aggregate +# Step 11: for each meter in list, get energy input data from energy database +# Step 12: for each virtual meter in list, get energy input data from energy database +# Step 13: for each offline meter in list, get energy input data from energy database +# Step 14: for each combined equipment in list, get energy input data from energy database +# Step 15: for each equipment in list, get energy input data from energy database +# Step 16: for each shopfloor in list, get energy input data from energy database +# Step 17: for each store in list, get energy input data from energy database +# Step 18: for each tenant in list, get energy input data from energy database +# Step 19: for each child space in list, get energy input data from energy database +# Step 20: determine common time slot to aggregate +# Step 21: aggregate energy data in the common time slot by energy categories and hourly +# Step 22: save energy data to energy database +# +# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter +######################################################################################################################## + +def worker(space): + #################################################################################################################### + # Step 1: get all input meters associated with the space + #################################################################################################################### + print("Step 1: get all input meters associated with the space " + str(space['name'])) + + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + error_string = "Error in step 1.1 of space_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + meter_list = list() + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_meters m, tbl_spaces_meters sm " + " WHERE m.id = sm.meter_id " + " AND m.is_counted = true " + " AND sm.space_id = %s ", + (space['id'],)) + rows_meters = cursor_system_db.fetchall() + + if rows_meters is not None and len(rows_meters) > 0: + for row in rows_meters: + meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 1.2 of space_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 2: get all input virtual meters associated with the space + #################################################################################################################### + print("Step 2: get all input virtual meters associated with the space") + virtual_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_virtual_meters m, tbl_spaces_virtual_meters sm " + " WHERE m.id = sm.virtual_meter_id " + " AND m.is_counted = true " + " AND sm.space_id = %s ", + (space['id'],)) + rows_virtual_meters = cursor_system_db.fetchall() + + if rows_virtual_meters is not None and len(rows_virtual_meters) > 0: + for row in rows_virtual_meters: + virtual_meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 2 of space_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 3: get all input offline meters associated with the space + #################################################################################################################### + print("Step 3: get all input offline meters associated with the space") + + offline_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_offline_meters m, tbl_spaces_offline_meters sm " + " WHERE m.id = sm.offline_meter_id " + " AND m.is_counted = true " + " AND sm.space_id = %s ", + (space['id'],)) + rows_offline_meters = cursor_system_db.fetchall() + + if rows_offline_meters is not None and len(rows_offline_meters) > 0: + for row in rows_offline_meters: + offline_meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 3 of space_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 4: get all combined equipments associated with the space + #################################################################################################################### + print("Step 4: get all combined equipments associated with the space") + + combined_equipment_list = list() + + try: + cursor_system_db.execute(" SELECT e.id, e.name " + " FROM tbl_combined_equipments e, tbl_spaces_combined_equipments se " + " WHERE e.id = se.combined_equipment_id " + " AND e.is_input_counted = true " + " AND se.space_id = %s ", + (space['id'],)) + rows_combined_equipments = cursor_system_db.fetchall() + + if rows_combined_equipments is not None and len(rows_combined_equipments) > 0: + for row in rows_combined_equipments: + combined_equipment_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 4 of space_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 5: get all equipments associated with the space + #################################################################################################################### + print("Step 5: get all equipments associated with the space") + + equipment_list = list() + + try: + cursor_system_db.execute(" SELECT e.id, e.name " + " FROM tbl_equipments e, tbl_spaces_equipments se " + " WHERE e.id = se.equipment_id " + " AND e.is_input_counted = true " + " AND se.space_id = %s ", + (space['id'],)) + rows_equipments = cursor_system_db.fetchall() + + if rows_equipments is not None and len(rows_equipments) > 0: + for row in rows_equipments: + equipment_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 5 of space_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 6: get all shopfloors associated with the space + #################################################################################################################### + print("Step 6: get all shopfloors associated with the space") + + shopfloor_list = list() + + try: + cursor_system_db.execute(" SELECT s.id, s.name " + " FROM tbl_shopfloors s, tbl_spaces_shopfloors ss " + " WHERE s.id = ss.shopfloor_id " + " AND s.is_input_counted = true " + " AND ss.space_id = %s ", + (space['id'],)) + rows_shopfloors = cursor_system_db.fetchall() + + if rows_shopfloors is not None and len(rows_shopfloors) > 0: + for row in rows_shopfloors: + shopfloor_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 6 of space_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 7: get all stores associated with the space + #################################################################################################################### + print("Step 7: get all stores associated with the space") + + store_list = list() + + try: + cursor_system_db.execute(" SELECT s.id, s.name " + " FROM tbl_stores s, tbl_spaces_stores ss " + " WHERE s.id = ss.store_id " + " AND s.is_input_counted = true " + " AND ss.space_id = %s ", + (space['id'],)) + rows_stores = cursor_system_db.fetchall() + + if rows_stores is not None and len(rows_stores) > 0: + for row in rows_stores: + store_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 7 of space_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 8: get all tenants associated with the space + #################################################################################################################### + print("Step 8: get all tenants associated with the space") + + tenant_list = list() + + try: + cursor_system_db.execute(" SELECT t.id, t.name " + " FROM tbl_tenants t, tbl_spaces_tenants st " + " WHERE t.id = st.tenant_id " + " AND t.is_input_counted = true " + " AND st.space_id = %s ", + (space['id'],)) + rows_tenants = cursor_system_db.fetchall() + + if rows_tenants is not None and len(rows_tenants) > 0: + for row in rows_tenants: + tenant_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 8 of space_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 9: get all child spaces associated with the space + #################################################################################################################### + print("Step 9: get all child spaces associated with the space") + + child_space_list = list() + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_spaces " + " WHERE is_input_counted = true " + " AND parent_space_id = %s ", + (space['id'],)) + rows_child_spaces = cursor_system_db.fetchall() + + if rows_child_spaces is not None and len(rows_child_spaces) > 0: + for row in rows_child_spaces: + child_space_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 9 of space_energy_input_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + if (meter_list is None or len(meter_list) == 0) and \ + (virtual_meter_list is None or len(virtual_meter_list) == 0) and \ + (offline_meter_list is None or len(offline_meter_list) == 0) and \ + (combined_equipment_list is None or len(combined_equipment_list) == 0) and \ + (equipment_list is None or len(equipment_list) == 0) and \ + (shopfloor_list is None or len(shopfloor_list) == 0) and \ + (store_list is None or len(store_list) == 0) and \ + (tenant_list is None or len(tenant_list) == 0) and \ + (child_space_list is None or len(child_space_list) == 0): + print("This is an empty space ") + return None + + #################################################################################################################### + # Step 10: determine start datetime and end datetime to aggregate + #################################################################################################################### + print("Step 10: determine start datetime and end datetime to aggregate") + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + error_string = "Error in step 10.1 of space_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + try: + query = (" SELECT MAX(start_datetime_utc) " + " FROM tbl_space_input_category_hourly " + " WHERE space_id = %s ") + cursor_energy_db.execute(query, (space['id'],)) + row_datetime = cursor_energy_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19] + + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19]) + + except Exception as e: + error_string = "Error in step 10.2 of space_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 11: for each meter in list, get energy input data from energy database + #################################################################################################################### + energy_meter_hourly = dict() + try: + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_meter_hourly " + " WHERE meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_meter_hourly[meter_id] = None + else: + energy_meter_hourly[meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 11 of space_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 12: for each virtual meter in list, get energy input data from energy database + #################################################################################################################### + energy_virtual_meter_hourly = dict() + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + try: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_virtual_meter_hourly " + " WHERE virtual_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_virtual_meter_hourly[virtual_meter_id] = None + else: + energy_virtual_meter_hourly[virtual_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 12 of space_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 13: for each offline meter in list, get energy input data from energy database + #################################################################################################################### + energy_offline_meter_hourly = dict() + if offline_meter_list is not None and len(offline_meter_list) > 0: + try: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_offline_meter_hourly " + " WHERE offline_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_offline_meter_hourly[offline_meter_id] = None + else: + energy_offline_meter_hourly[offline_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1] + + except Exception as e: + error_string = "Error in step 13 of space_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 14: for each combined equipment in list, get energy input data from energy database + #################################################################################################################### + energy_combined_equipment_hourly = dict() + if combined_equipment_list is not None and len(combined_equipment_list) > 0: + try: + for combined_equipment in combined_equipment_list: + combined_equipment_id = str(combined_equipment['id']) + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_combined_equipment_input_category_hourly " + " WHERE combined_equipment_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (combined_equipment_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_combined_equipment_hourly[combined_equipment_id] = None + else: + energy_combined_equipment_hourly[combined_equipment_id] = dict() + for row_value in rows_energy_values: + current_datetime_utc = row_value[0] + if current_datetime_utc not in energy_combined_equipment_hourly[combined_equipment_id]: + energy_combined_equipment_hourly[combined_equipment_id][current_datetime_utc] = dict() + energy_category_id = row_value[1] + actual_value = row_value[2] + energy_combined_equipment_hourly[combined_equipment_id][current_datetime_utc][energy_category_id] = actual_value + except Exception as e: + error_string = "Error in step 14 of space_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 15: for each equipment in list, get energy input data from energy database + #################################################################################################################### + energy_equipment_hourly = dict() + if equipment_list is not None and len(equipment_list) > 0: + try: + for equipment in equipment_list: + equipment_id = str(equipment['id']) + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_equipment_input_category_hourly " + " WHERE equipment_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (equipment_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_equipment_hourly[equipment_id] = None + else: + energy_equipment_hourly[equipment_id] = dict() + for row_value in rows_energy_values: + current_datetime_utc = row_value[0] + if current_datetime_utc not in energy_equipment_hourly[equipment_id]: + energy_equipment_hourly[equipment_id][current_datetime_utc] = dict() + energy_category_id = row_value[1] + actual_value = row_value[2] + energy_equipment_hourly[equipment_id][current_datetime_utc][energy_category_id] = \ + actual_value + except Exception as e: + error_string = "Error in step 15 of space_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 16: for each shopfloor in list, get energy input data from energy database + #################################################################################################################### + energy_shopfloor_hourly = dict() + if shopfloor_list is not None and len(shopfloor_list) > 0: + try: + for shopfloor in shopfloor_list: + shopfloor_id = str(shopfloor['id']) + + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_shopfloor_input_category_hourly " + " WHERE shopfloor_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (shopfloor_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_shopfloor_hourly[shopfloor_id] = None + else: + energy_shopfloor_hourly[shopfloor_id] = dict() + for row_energy_value in rows_energy_values: + current_datetime_utc = row_energy_value[0] + if current_datetime_utc not in energy_shopfloor_hourly[shopfloor_id]: + energy_shopfloor_hourly[shopfloor_id][current_datetime_utc] = dict() + energy_category_id = row_energy_value[1] + actual_value = row_energy_value[2] + energy_shopfloor_hourly[shopfloor_id][current_datetime_utc][energy_category_id] = actual_value + except Exception as e: + error_string = "Error in step 16 of space_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 17: for each store in list, get energy input data from energy database + #################################################################################################################### + energy_store_hourly = dict() + if store_list is not None and len(store_list) > 0: + try: + for store in store_list: + store_id = str(store['id']) + + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_store_input_category_hourly " + " WHERE store_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (store_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_store_hourly[store_id] = None + else: + energy_store_hourly[store_id] = dict() + for row_energy_value in rows_energy_values: + current_datetime_utc = row_energy_value[0] + if current_datetime_utc not in energy_store_hourly[store_id]: + energy_store_hourly[store_id][current_datetime_utc] = dict() + energy_category_id = row_energy_value[1] + actual_value = row_energy_value[2] + energy_store_hourly[store_id][current_datetime_utc][energy_category_id] = actual_value + except Exception as e: + error_string = "Error in step 17 of space_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 18: for each tenant in list, get energy input data from energy database + #################################################################################################################### + energy_tenant_hourly = dict() + if tenant_list is not None and len(tenant_list) > 0: + try: + for tenant in tenant_list: + tenant_id = str(tenant['id']) + + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_tenant_input_category_hourly " + " WHERE tenant_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (tenant_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_tenant_hourly[tenant_id] = None + else: + energy_tenant_hourly[tenant_id] = dict() + for row_energy_value in rows_energy_values: + current_datetime_utc = row_energy_value[0] + if current_datetime_utc not in energy_tenant_hourly[tenant_id]: + energy_tenant_hourly[tenant_id][current_datetime_utc] = dict() + energy_category_id = row_energy_value[1] + actual_value = row_energy_value[2] + energy_tenant_hourly[tenant_id][current_datetime_utc][energy_category_id] = actual_value + except Exception as e: + error_string = "Error in step 18 of space_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 19: for each child space in list, get energy input data from energy database + #################################################################################################################### + energy_child_space_hourly = dict() + if child_space_list is not None and len(child_space_list) > 0: + try: + for child_space in child_space_list: + child_space_id = str(child_space['id']) + + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_space_input_category_hourly " + " WHERE space_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (child_space_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_child_space_hourly[child_space_id] = None + else: + energy_child_space_hourly[child_space_id] = dict() + for row_energy_value in rows_energy_values: + current_datetime_utc = row_energy_value[0] + if current_datetime_utc not in energy_child_space_hourly[child_space_id]: + energy_child_space_hourly[child_space_id][current_datetime_utc] = dict() + energy_category_id = row_energy_value[1] + actual_value = row_energy_value[2] + energy_child_space_hourly[child_space_id][current_datetime_utc][energy_category_id] = actual_value + except Exception as e: + error_string = "Error in step 19 of space_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 20: determine common time slot to aggregate + #################################################################################################################### + + common_start_datetime_utc = start_datetime_utc + common_end_datetime_utc = end_datetime_utc + + print("Getting common time slot of energy values for all meters") + if energy_meter_hourly is not None and len(energy_meter_hourly) > 0: + for meter_id, energy_hourly in energy_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all virtual meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0: + for meter_id, energy_hourly in energy_virtual_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all offline meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0: + for meter_id, energy_hourly in energy_offline_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all combined equipments") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_combined_equipment_hourly is not None and len(energy_combined_equipment_hourly) > 0: + for combined_equipment_id, energy_hourly in energy_combined_equipment_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all equipments") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_equipment_hourly is not None and len(energy_equipment_hourly) > 0: + for equipment_id, energy_hourly in energy_equipment_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all shopfloors") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_shopfloor_hourly is not None and len(energy_shopfloor_hourly) > 0: + for shopfloor_id, energy_hourly in energy_shopfloor_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all stores") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_store_hourly is not None and len(energy_store_hourly) > 0: + for store_id, energy_hourly in energy_store_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all tenants") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_tenant_hourly is not None and len(energy_tenant_hourly) > 0: + for tenant_id, energy_hourly in energy_tenant_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all child spaces") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_child_space_hourly is not None and len(energy_child_space_hourly) > 0: + for child_space_id, energy_hourly in energy_child_space_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \ + (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \ + (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0) and \ + (energy_combined_equipment_hourly is None or len(energy_combined_equipment_hourly) == 0) and \ + (energy_equipment_hourly is None or len(energy_equipment_hourly) == 0) and \ + (energy_shopfloor_hourly is None or len(energy_shopfloor_hourly) == 0) and \ + (energy_store_hourly is None or len(energy_store_hourly) == 0) and \ + (energy_tenant_hourly is None or len(energy_tenant_hourly) == 0) and \ + (energy_child_space_hourly is None or len(energy_child_space_hourly) == 0): + # There isn't any energy data + print("There isn't any energy data") + # continue the for space loop to the next space + print("continue the for space loop to the next space") + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + return None + + print("common_start_datetime_utc: " + str(common_start_datetime_utc)) + print("common_end_datetime_utc: " + str(common_end_datetime_utc)) + + #################################################################################################################### + # Step 21: aggregate energy data in the common time slot by energy categories and hourly + #################################################################################################################### + + print("Step 21: aggregate energy data in the common time slot by energy categories and hourly") + aggregated_values = list() + try: + current_datetime_utc = common_start_datetime_utc + while common_start_datetime_utc is not None \ + and common_end_datetime_utc is not None \ + and current_datetime_utc <= common_end_datetime_utc: + aggregated_value = dict() + aggregated_value['start_datetime_utc'] = current_datetime_utc + aggregated_value['meta_data'] = dict() + + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + energy_category_id = meter['energy_category_id'] + actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + energy_category_id = virtual_meter['energy_category_id'] + actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if offline_meter_list is not None and len(offline_meter_list) > 0: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + energy_category_id = offline_meter['energy_category_id'] + actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if combined_equipment_list is not None and len(combined_equipment_list) > 0: + for combined_equipment in combined_equipment_list: + combined_equipment_id = str(combined_equipment['id']) + meta_data_dict = \ + energy_combined_equipment_hourly[combined_equipment_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_category_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if equipment_list is not None and len(equipment_list) > 0: + for equipment in equipment_list: + equipment_id = str(equipment['id']) + meta_data_dict = energy_equipment_hourly[equipment_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_category_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if shopfloor_list is not None and len(shopfloor_list) > 0: + for shopfloor in shopfloor_list: + shopfloor_id = str(shopfloor['id']) + meta_data_dict = energy_shopfloor_hourly[shopfloor_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_category_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if store_list is not None and len(store_list) > 0: + for store in store_list: + store_id = str(store['id']) + meta_data_dict = energy_store_hourly[store_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_category_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if tenant_list is not None and len(tenant_list) > 0: + for tenant in tenant_list: + tenant_id = str(tenant['id']) + meta_data_dict = energy_tenant_hourly[tenant_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_category_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if child_space_list is not None and len(child_space_list) > 0: + for child_space in child_space_list: + child_space_id = str(child_space['id']) + meta_data_dict = energy_child_space_hourly[child_space_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_category_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + aggregated_values.append(aggregated_value) + + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + except Exception as e: + error_string = "Error in step 21 of space_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 22: save energy data to energy database + #################################################################################################################### + print("Step 22: save energy data to energy database") + + if len(aggregated_values) > 0: + try: + add_values = (" INSERT INTO tbl_space_input_category_hourly " + " (space_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for aggregated_value in aggregated_values: + for energy_category_id, actual_value in aggregated_value['meta_data'].items(): + add_values += " (" + str(space['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "'," + add_values += str(actual_value) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_energy_db.execute(add_values[:-2]) + cnx_energy_db.commit() + + except Exception as e: + error_string = "Error in step 22 of space_energy_input_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + else: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() diff --git a/myems-aggregation/space_energy_input_item.py b/myems-aggregation/space_energy_input_item.py new file mode 100644 index 00000000..4589897e --- /dev/null +++ b/myems-aggregation/space_energy_input_item.py @@ -0,0 +1,1084 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +from multiprocessing import Pool +import random +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all spaces +# Step 2: Create multiprocessing pool to call worker in parallel +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all spaces + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of space_energy_input_item.main " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_spaces " + " ORDER BY id ") + rows_spaces = cursor_system_db.fetchall() + + if rows_spaces is None or len(rows_spaces) == 0: + print("There isn't any spaces ") + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + + space_list = list() + for row in rows_spaces: + space_list.append({"id": row[0], "name": row[1]}) + + except Exception as e: + logger.error("Error in step 1.2 of space_energy_input_item.main " + str(e)) + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + print("Got all spaces in MyEMS System Database") + + # shuffle the space list for randomly calculating the meter hourly value + random.shuffle(space_list) + + ################################################################################################################ + # Step 2: Create multiprocessing pool to call worker in parallel + ################################################################################################################ + p = Pool(processes=config.pool_size) + error_list = p.map(worker, space_list) + p.close() + p.join() + + for error in error_list: + if error is not None and len(error) > 0: + logger.error(error) + + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of outer while + + +######################################################################################################################## +# PROCEDURES: +# Step 1: get all input meters associated with the space +# Step 2: get all input virtual meters associated with the space +# Step 3: get all input offline meters associated with the space +# Step 4: get all combined equipments associated with the space +# Step 5: get all equipments associated with the space +# Step 6: get all shopfloors associated with the space +# Step 7: get all stores associated with the space +# Step 8: get all tenants associated with the space +# Step 9: get all child spaces associated with the space +# Step 10: determine start datetime and end datetime to aggregate +# Step 11: for each meter in list, get energy input data from energy database +# Step 12: for each virtual meter in list, get energy input data from energy database +# Step 13: for each offline meter in list, get energy input data from energy database +# Step 14: for each combined equipment in list, get energy input data from energy database +# Step 15: for each equipment in list, get energy input data from energy database +# Step 16: for each shopfloor in list, get energy input data from energy database +# Step 17: for each store in list, get energy input data from energy database +# Step 18: for each tenant in list, get energy input data from energy database +# Step 19: for each child space in list, get energy input data from energy database +# Step 20: determine common time slot to aggregate +# Step 21: aggregate energy data in the common time slot by energy items and hourly +# Step 22: save energy data to energy database +# +# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter +######################################################################################################################## + +def worker(space): + #################################################################################################################### + # Step 1: get all input meters associated with the space + #################################################################################################################### + print("Step 1: get all input meters associated with the space " + str(space['name'])) + + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + error_string = "Error in step 1.1 of space_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + meter_list = list() + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_meters m, tbl_spaces_meters sm " + " WHERE m.id = sm.meter_id " + " AND m.is_counted = true " + " AND m.energy_item_id is NOT NULL " + " AND sm.space_id = %s ", + (space['id'],)) + rows_meters = cursor_system_db.fetchall() + + if rows_meters is not None and len(rows_meters) > 0: + for row in rows_meters: + meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 1.2 of space_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 2: get all input virtual meters associated with the space + #################################################################################################################### + print("Step 2: get all input virtual meters associated with the space") + virtual_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_virtual_meters m, tbl_spaces_virtual_meters sm " + " WHERE m.id = sm.virtual_meter_id " + " AND m.is_counted = true " + " AND m.energy_item_id is NOT NULL " + " AND sm.space_id = %s ", + (space['id'],)) + rows_virtual_meters = cursor_system_db.fetchall() + + if rows_virtual_meters is not None and len(rows_virtual_meters) > 0: + for row in rows_virtual_meters: + virtual_meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 2 of space_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 3: get all input offline meters associated with the space + #################################################################################################################### + print("Step 3: get all input offline meters associated with the space") + + offline_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_offline_meters m, tbl_spaces_offline_meters sm " + " WHERE m.id = sm.offline_meter_id " + " AND m.is_counted = true " + " AND m.energy_item_id is NOT NULL " + " AND sm.space_id = %s ", + (space['id'],)) + rows_offline_meters = cursor_system_db.fetchall() + + if rows_offline_meters is not None and len(rows_offline_meters) > 0: + for row in rows_offline_meters: + offline_meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 3 of space_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 4: get all combined equipments associated with the space + #################################################################################################################### + print("Step 4: get all combined equipments associated with the space") + + combined_equipment_list = list() + + try: + cursor_system_db.execute(" SELECT e.id, e.name " + " FROM tbl_combined_equipments e, tbl_spaces_combined_equipments se " + " WHERE e.id = se.combined_equipment_id " + " AND e.is_input_counted = true " + " AND se.space_id = %s ", + (space['id'],)) + rows_combined_equipments = cursor_system_db.fetchall() + + if rows_combined_equipments is not None and len(rows_combined_equipments) > 0: + for row in rows_combined_equipments: + combined_equipment_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 4 of space_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 5: get all equipments associated with the space + #################################################################################################################### + print("Step 5: get all equipments associated with the space") + + equipment_list = list() + + try: + cursor_system_db.execute(" SELECT e.id, e.name " + " FROM tbl_equipments e, tbl_spaces_equipments se " + " WHERE e.id = se.equipment_id " + " AND e.is_input_counted = true " + " AND se.space_id = %s ", + (space['id'],)) + rows_equipments = cursor_system_db.fetchall() + + if rows_equipments is not None and len(rows_equipments) > 0: + for row in rows_equipments: + equipment_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 5 of space_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 6: get all shopfloors associated with the space + #################################################################################################################### + print("Step 6: get all shopfloors associated with the space") + + shopfloor_list = list() + + try: + cursor_system_db.execute(" SELECT s.id, s.name " + " FROM tbl_shopfloors s, tbl_spaces_shopfloors ss " + " WHERE s.id = ss.shopfloor_id " + " AND s.is_input_counted = true " + " AND ss.space_id = %s ", + (space['id'],)) + rows_shopfloors = cursor_system_db.fetchall() + + if rows_shopfloors is not None and len(rows_shopfloors) > 0: + for row in rows_shopfloors: + shopfloor_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 6 of space_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 7: get all stores associated with the space + #################################################################################################################### + print("Step 7: get all stores associated with the space") + + store_list = list() + + try: + cursor_system_db.execute(" SELECT s.id, s.name " + " FROM tbl_stores s, tbl_spaces_stores ss " + " WHERE s.id = ss.store_id " + " AND s.is_input_counted = true " + " AND ss.space_id = %s ", + (space['id'],)) + rows_stores = cursor_system_db.fetchall() + + if rows_stores is not None and len(rows_stores) > 0: + for row in rows_stores: + store_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 7 of space_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 8: get all tenants associated with the space + #################################################################################################################### + print("Step 8: get all tenants associated with the space") + + tenant_list = list() + + try: + cursor_system_db.execute(" SELECT t.id, t.name " + " FROM tbl_tenants t, tbl_spaces_tenants st " + " WHERE t.id = st.tenant_id " + " AND t.is_input_counted = true " + " AND st.space_id = %s ", + (space['id'],)) + rows_tenants = cursor_system_db.fetchall() + + if rows_tenants is not None and len(rows_tenants) > 0: + for row in rows_tenants: + tenant_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 8 of space_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 9: get all child spaces associated with the space + #################################################################################################################### + print("Step 9: get all child spaces associated with the space") + + child_space_list = list() + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_spaces " + " WHERE is_input_counted = true " + " AND parent_space_id = %s ", + (space['id'],)) + rows_child_spaces = cursor_system_db.fetchall() + + if rows_child_spaces is not None and len(rows_child_spaces) > 0: + for row in rows_child_spaces: + child_space_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 9 of space_energy_input_item.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + if (meter_list is None or len(meter_list) == 0) and \ + (virtual_meter_list is None or len(virtual_meter_list) == 0) and \ + (offline_meter_list is None or len(offline_meter_list) == 0) and \ + (combined_equipment_list is None or len(combined_equipment_list) == 0) and \ + (equipment_list is None or len(equipment_list) == 0) and \ + (shopfloor_list is None or len(shopfloor_list) == 0) and \ + (store_list is None or len(store_list) == 0) and \ + (tenant_list is None or len(tenant_list) == 0) and \ + (child_space_list is None or len(child_space_list) == 0): + print("This is an empty space ") + return None + + #################################################################################################################### + # Step 10: determine start datetime and end datetime to aggregate + #################################################################################################################### + print("Step 10: determine start datetime and end datetime to aggregate") + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + error_string = "Error in step 10.1 of space_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + try: + query = (" SELECT MAX(start_datetime_utc) " + " FROM tbl_space_input_item_hourly " + " WHERE space_id = %s ") + cursor_energy_db.execute(query, (space['id'],)) + row_datetime = cursor_energy_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19] + + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19]) + + except Exception as e: + error_string = "Error in step 10.2 of space_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 11: for each meter in list, get energy input data from energy database + #################################################################################################################### + energy_meter_hourly = dict() + try: + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_meter_hourly " + " WHERE meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_meter_hourly[meter_id] = None + else: + energy_meter_hourly[meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 11 of space_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 12: for each virtual meter in list, get energy input data from energy database + #################################################################################################################### + energy_virtual_meter_hourly = dict() + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + try: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_virtual_meter_hourly " + " WHERE virtual_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_virtual_meter_hourly[virtual_meter_id] = None + else: + energy_virtual_meter_hourly[virtual_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 12 of space_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 13: for each offline meter in list, get energy input data from energy database + #################################################################################################################### + energy_offline_meter_hourly = dict() + if offline_meter_list is not None and len(offline_meter_list) > 0: + try: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_offline_meter_hourly " + " WHERE offline_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_offline_meter_hourly[offline_meter_id] = None + else: + energy_offline_meter_hourly[offline_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1] + + except Exception as e: + error_string = "Error in step 13 of space_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 14: for each combined equipment in list, get energy input data from energy database + #################################################################################################################### + energy_combined_equipment_hourly = dict() + if combined_equipment_list is not None and len(combined_equipment_list) > 0: + try: + for combined_equipment in combined_equipment_list: + combined_equipment_id = str(combined_equipment['id']) + query = (" SELECT start_datetime_utc, energy_item_id, actual_value " + " FROM tbl_combined_equipment_input_item_hourly " + " WHERE combined_equipment_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (combined_equipment_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_combined_equipment_hourly[combined_equipment_id] = None + else: + energy_combined_equipment_hourly[combined_equipment_id] = dict() + for row_value in rows_energy_values: + current_datetime_utc = row_value[0] + if current_datetime_utc not in energy_combined_equipment_hourly[combined_equipment_id]: + energy_combined_equipment_hourly[combined_equipment_id][current_datetime_utc] = dict() + energy_item_id = row_value[1] + actual_value = row_value[2] + energy_combined_equipment_hourly[combined_equipment_id][current_datetime_utc][energy_item_id] = actual_value + except Exception as e: + error_string = "Error in step 14 of space_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 15: for each equipment in list, get energy input data from energy database + #################################################################################################################### + energy_equipment_hourly = dict() + if equipment_list is not None and len(equipment_list) > 0: + try: + for equipment in equipment_list: + equipment_id = str(equipment['id']) + query = (" SELECT start_datetime_utc, energy_item_id, actual_value " + " FROM tbl_equipment_input_item_hourly " + " WHERE equipment_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (equipment_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_equipment_hourly[equipment_id] = None + else: + energy_equipment_hourly[equipment_id] = dict() + for row_value in rows_energy_values: + current_datetime_utc = row_value[0] + if current_datetime_utc not in energy_equipment_hourly[equipment_id]: + energy_equipment_hourly[equipment_id][current_datetime_utc] = dict() + energy_item_id = row_value[1] + actual_value = row_value[2] + energy_equipment_hourly[equipment_id][current_datetime_utc][energy_item_id] = \ + actual_value + except Exception as e: + error_string = "Error in step 15 of space_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 16: for each shopfloor in list, get energy input data from energy database + #################################################################################################################### + energy_shopfloor_hourly = dict() + if shopfloor_list is not None and len(shopfloor_list) > 0: + try: + for shopfloor in shopfloor_list: + shopfloor_id = str(shopfloor['id']) + + query = (" SELECT start_datetime_utc, energy_item_id, actual_value " + " FROM tbl_shopfloor_input_item_hourly " + " WHERE shopfloor_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (shopfloor_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_shopfloor_hourly[shopfloor_id] = None + else: + energy_shopfloor_hourly[shopfloor_id] = dict() + for row_energy_value in rows_energy_values: + current_datetime_utc = row_energy_value[0] + if current_datetime_utc not in energy_shopfloor_hourly[shopfloor_id]: + energy_shopfloor_hourly[shopfloor_id][current_datetime_utc] = dict() + energy_item_id = row_energy_value[1] + actual_value = row_energy_value[2] + energy_shopfloor_hourly[shopfloor_id][current_datetime_utc][energy_item_id] = actual_value + except Exception as e: + error_string = "Error in step 16 of space_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 17: for each store in list, get energy input data from energy database + #################################################################################################################### + energy_store_hourly = dict() + if store_list is not None and len(store_list) > 0: + try: + for store in store_list: + store_id = str(store['id']) + + query = (" SELECT start_datetime_utc, energy_item_id, actual_value " + " FROM tbl_store_input_item_hourly " + " WHERE store_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (store_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_store_hourly[store_id] = None + else: + energy_store_hourly[store_id] = dict() + for row_energy_value in rows_energy_values: + current_datetime_utc = row_energy_value[0] + if current_datetime_utc not in energy_store_hourly[store_id]: + energy_store_hourly[store_id][current_datetime_utc] = dict() + energy_item_id = row_energy_value[1] + actual_value = row_energy_value[2] + energy_store_hourly[store_id][current_datetime_utc][energy_item_id] = actual_value + except Exception as e: + error_string = "Error in step 17 of space_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 18: for each tenant in list, get energy input data from energy database + #################################################################################################################### + energy_tenant_hourly = dict() + if tenant_list is not None and len(tenant_list) > 0: + try: + for tenant in tenant_list: + tenant_id = str(tenant['id']) + + query = (" SELECT start_datetime_utc, energy_item_id, actual_value " + " FROM tbl_tenant_input_item_hourly " + " WHERE tenant_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (tenant_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_tenant_hourly[tenant_id] = None + else: + energy_tenant_hourly[tenant_id] = dict() + for row_energy_value in rows_energy_values: + current_datetime_utc = row_energy_value[0] + if current_datetime_utc not in energy_tenant_hourly[tenant_id]: + energy_tenant_hourly[tenant_id][current_datetime_utc] = dict() + energy_item_id = row_energy_value[1] + actual_value = row_energy_value[2] + energy_tenant_hourly[tenant_id][current_datetime_utc][energy_item_id] = actual_value + except Exception as e: + error_string = "Error in step 18 of space_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 19: for each child space in list, get energy input data from energy database + #################################################################################################################### + energy_child_space_hourly = dict() + if child_space_list is not None and len(child_space_list) > 0: + try: + for child_space in child_space_list: + child_space_id = str(child_space['id']) + + query = (" SELECT start_datetime_utc, energy_item_id, actual_value " + " FROM tbl_space_input_item_hourly " + " WHERE space_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (child_space_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_child_space_hourly[child_space_id] = None + else: + energy_child_space_hourly[child_space_id] = dict() + for row_energy_value in rows_energy_values: + current_datetime_utc = row_energy_value[0] + if current_datetime_utc not in energy_child_space_hourly[child_space_id]: + energy_child_space_hourly[child_space_id][current_datetime_utc] = dict() + energy_item_id = row_energy_value[1] + actual_value = row_energy_value[2] + energy_child_space_hourly[child_space_id][current_datetime_utc][energy_item_id] = actual_value + except Exception as e: + error_string = "Error in step 19 of space_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 20: determine common time slot to aggregate + #################################################################################################################### + + common_start_datetime_utc = start_datetime_utc + common_end_datetime_utc = end_datetime_utc + + print("Getting common time slot of energy values for all meters") + if energy_meter_hourly is not None and len(energy_meter_hourly) > 0: + for meter_id, energy_hourly in energy_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all virtual meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0: + for meter_id, energy_hourly in energy_virtual_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all offline meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0: + for meter_id, energy_hourly in energy_offline_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all combined equipments") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_combined_equipment_hourly is not None and len(energy_combined_equipment_hourly) > 0: + for combined_equipment_id, energy_hourly in energy_combined_equipment_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all equipments") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_equipment_hourly is not None and len(energy_equipment_hourly) > 0: + for equipment_id, energy_hourly in energy_equipment_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all shopfloors") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_shopfloor_hourly is not None and len(energy_shopfloor_hourly) > 0: + for shopfloor_id, energy_hourly in energy_shopfloor_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all stores") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_store_hourly is not None and len(energy_store_hourly) > 0: + for store_id, energy_hourly in energy_store_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all tenants") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_tenant_hourly is not None and len(energy_tenant_hourly) > 0: + for tenant_id, energy_hourly in energy_tenant_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all child spaces") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_child_space_hourly is not None and len(energy_child_space_hourly) > 0: + for child_space_id, energy_hourly in energy_child_space_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \ + (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \ + (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0) and \ + (energy_combined_equipment_hourly is None or len(energy_combined_equipment_hourly) == 0) and \ + (energy_equipment_hourly is None or len(energy_equipment_hourly) == 0) and \ + (energy_shopfloor_hourly is None or len(energy_shopfloor_hourly) == 0) and \ + (energy_store_hourly is None or len(energy_store_hourly) == 0) and \ + (energy_tenant_hourly is None or len(energy_tenant_hourly) == 0) and \ + (energy_child_space_hourly is None or len(energy_child_space_hourly) == 0): + # There isn't any energy data + print("There isn't any energy data") + # continue the for space loop to the next space + print("continue the for space loop to the next space") + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + return None + + print("common_start_datetime_utc: " + str(common_start_datetime_utc)) + print("common_end_datetime_utc: " + str(common_end_datetime_utc)) + + #################################################################################################################### + # Step 21: aggregate energy data in the common time slot by energy items and hourly + #################################################################################################################### + + print("Step 21: aggregate energy data in the common time slot by energy items and hourly") + aggregated_values = list() + try: + current_datetime_utc = common_start_datetime_utc + while common_start_datetime_utc is not None \ + and common_end_datetime_utc is not None \ + and current_datetime_utc <= common_end_datetime_utc: + aggregated_value = dict() + aggregated_value['start_datetime_utc'] = current_datetime_utc + aggregated_value['meta_data'] = dict() + + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + energy_item_id = meter['energy_item_id'] + actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + energy_item_id = virtual_meter['energy_item_id'] + actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if offline_meter_list is not None and len(offline_meter_list) > 0: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + energy_item_id = offline_meter['energy_item_id'] + actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if combined_equipment_list is not None and len(combined_equipment_list) > 0: + for combined_equipment in combined_equipment_list: + combined_equipment_id = str(combined_equipment['id']) + meta_data_dict = \ + energy_combined_equipment_hourly[combined_equipment_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_item_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if equipment_list is not None and len(equipment_list) > 0: + for equipment in equipment_list: + equipment_id = str(equipment['id']) + meta_data_dict = energy_equipment_hourly[equipment_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_item_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if shopfloor_list is not None and len(shopfloor_list) > 0: + for shopfloor in shopfloor_list: + shopfloor_id = str(shopfloor['id']) + meta_data_dict = energy_store_hourly[shopfloor_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_item_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if store_list is not None and len(store_list) > 0: + for store in store_list: + store_id = str(store['id']) + meta_data_dict = energy_store_hourly[store_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_item_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if tenant_list is not None and len(tenant_list) > 0: + for tenant in tenant_list: + tenant_id = str(tenant['id']) + meta_data_dict = energy_tenant_hourly[tenant_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_item_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if child_space_list is not None and len(child_space_list) > 0: + for child_space in child_space_list: + child_space_id = str(child_space['id']) + meta_data_dict = energy_child_space_hourly[child_space_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_item_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + aggregated_values.append(aggregated_value) + + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + except Exception as e: + error_string = "Error in step 19 of space_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 22: save energy data to energy database + #################################################################################################################### + print("Step 22: save energy data to energy database") + + if len(aggregated_values) > 0: + try: + add_values = (" INSERT INTO tbl_space_input_item_hourly " + " (space_id, " + " energy_item_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for aggregated_value in aggregated_values: + for energy_item_id, actual_value in aggregated_value['meta_data'].items(): + add_values += " (" + str(space['id']) + "," + add_values += " " + str(energy_item_id) + "," + add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "'," + add_values += str(actual_value) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_energy_db.execute(add_values[:-2]) + cnx_energy_db.commit() + + except Exception as e: + error_string = "Error in step 20 of space_energy_input_item.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + else: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() diff --git a/myems-aggregation/space_energy_output_category.py b/myems-aggregation/space_energy_output_category.py new file mode 100644 index 00000000..24cec572 --- /dev/null +++ b/myems-aggregation/space_energy_output_category.py @@ -0,0 +1,533 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +from multiprocessing import Pool +import random +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all spaces +# Step 2: Create multiprocessing pool to call worker in parallel +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all spaces + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of space_energy_output_category.main " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_spaces " + " ORDER BY id ") + rows_spaces = cursor_system_db.fetchall() + + if rows_spaces is None or len(rows_spaces) == 0: + print("There isn't any spaces ") + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + + space_list = list() + for row in rows_spaces: + space_list.append({"id": row[0], "name": row[1]}) + + except Exception as e: + logger.error("Error in step 1.2 of space_energy_output_category.main " + str(e)) + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + print("Got all spaces in MyEMS System Database") + + # shuffle the space list for randomly calculating the meter hourly value + random.shuffle(space_list) + + ################################################################################################################ + # Step 2: Create multiprocessing pool to call worker in parallel + ################################################################################################################ + p = Pool(processes=config.pool_size) + error_list = p.map(worker, space_list) + p.close() + p.join() + + for error in error_list: + if error is not None and len(error) > 0: + logger.error(error) + + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of outer while + + +######################################################################################################################## +# PROCEDURES: +# Step 1: get all combined equipments associated with the space +# Step 2: get all equipments associated with the space +# Step 3: get all child spaces associated with the space +# Step 4: determine start datetime and end datetime to aggregate +# Step 5: for each combined equipment in list, get energy output data from energy database +# Step 6: for each equipment in list, get energy output data from energy database +# Step 7: for each child space in list, get energy output data from energy database +# Step 8: determine common time slot to aggregate +# Step 9: aggregate energy data in the common time slot by energy categories and hourly +# Step 10: save energy data to energy database +# +# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter +######################################################################################################################## + +def worker(space): + + #################################################################################################################### + # Step 1: get all combined equipments associated with the space + #################################################################################################################### + print("Step 1: get all combined equipments associated with the space") + + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + error_string = "Error in step 1.1 of space_energy_output_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + combined_equipment_list = list() + try: + cursor_system_db.execute(" SELECT e.id, e.name " + " FROM tbl_combined_equipments e, tbl_spaces_combined_equipments se " + " WHERE e.id = se.combined_equipment_id " + " AND e.is_output_counted = true " + " AND se.space_id = %s ", + (space['id'],)) + rows_combined_equipments = cursor_system_db.fetchall() + + if rows_combined_equipments is not None and len(rows_combined_equipments) > 0: + for row in rows_combined_equipments: + combined_equipment_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 1.2 of space_energy_output_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 2: get all equipments associated with the space + #################################################################################################################### + print("Step 2: get all equipments associated with the space") + + equipment_list = list() + try: + cursor_system_db.execute(" SELECT e.id, e.name " + " FROM tbl_equipments e, tbl_spaces_equipments se " + " WHERE e.id = se.equipment_id " + " AND e.is_output_counted = true " + " AND se.space_id = %s ", + (space['id'],)) + rows_equipments = cursor_system_db.fetchall() + + if rows_equipments is not None and len(rows_equipments) > 0: + for row in rows_equipments: + equipment_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 2.2 of space_energy_output_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 3: get all child spaces associated with the space + #################################################################################################################### + print("Step 3: get all child spaces associated with the space") + + child_space_list = list() + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_spaces " + " WHERE is_output_counted = true " + " AND parent_space_id = %s ", + (space['id'],)) + rows_child_spaces = cursor_system_db.fetchall() + + if rows_child_spaces is not None and len(rows_child_spaces) > 0: + for row in rows_child_spaces: + child_space_list.append({"id": row[0], + "name": row[1]}) + + except Exception as e: + error_string = "Error in step 3 of space_energy_output_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + if ((combined_equipment_list is None or len(combined_equipment_list) == 0) and + (equipment_list is None or len(equipment_list) == 0) and + (child_space_list is None or len(child_space_list) == 0)): + print("This is an empty space ") + return None + + #################################################################################################################### + # Step 4: determine start datetime and end datetime to aggregate + #################################################################################################################### + print("Step 4: determine start datetime and end datetime to aggregate") + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + error_string = "Error in step 4.1 of space_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + try: + query = (" SELECT MAX(start_datetime_utc) " + " FROM tbl_space_output_category_hourly " + " WHERE space_id = %s ") + cursor_energy_db.execute(query, (space['id'],)) + row_datetime = cursor_energy_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19] + + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19]) + + except Exception as e: + error_string = "Error in step 4.2 of space_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 5: for each combined equipment in list, get energy output data from energy database + #################################################################################################################### + energy_combined_equipment_hourly = dict() + if combined_equipment_list is not None and len(combined_equipment_list) > 0: + try: + for combined_equipment in combined_equipment_list: + combined_equipment_id = str(combined_equipment['id']) + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_combined_equipment_output_category_hourly " + " WHERE combined_equipment_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (combined_equipment_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_combined_equipment_hourly[combined_equipment_id] = None + else: + energy_combined_equipment_hourly[combined_equipment_id] = dict() + for row_value in rows_energy_values: + current_datetime_utc = row_value[0] + if current_datetime_utc not in energy_combined_equipment_hourly[combined_equipment_id]: + energy_combined_equipment_hourly[combined_equipment_id][current_datetime_utc] = dict() + energy_category_id = row_value[1] + actual_value = row_value[2] + energy_combined_equipment_hourly[combined_equipment_id][current_datetime_utc][energy_category_id] = \ + actual_value + except Exception as e: + error_string = "Error in step 5 of space_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 6: for each equipment in list, get energy output data from energy database + #################################################################################################################### + energy_equipment_hourly = dict() + if equipment_list is not None and len(equipment_list) > 0: + try: + for equipment in equipment_list: + equipment_id = str(equipment['id']) + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_equipment_output_category_hourly " + " WHERE equipment_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (equipment_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_equipment_hourly[equipment_id] = None + else: + energy_equipment_hourly[equipment_id] = dict() + for row_value in rows_energy_values: + current_datetime_utc = row_value[0] + if current_datetime_utc not in energy_equipment_hourly[equipment_id]: + energy_equipment_hourly[equipment_id][current_datetime_utc] = dict() + energy_category_id = row_value[1] + actual_value = row_value[2] + energy_equipment_hourly[equipment_id][current_datetime_utc][energy_category_id] = \ + actual_value + except Exception as e: + error_string = "Error in step 6 of space_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 7: for each child space in list, get energy output data from energy database + #################################################################################################################### + energy_child_space_hourly = dict() + if child_space_list is not None and len(child_space_list) > 0: + try: + for child_space in child_space_list: + child_space_id = str(child_space['id']) + + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_space_output_category_hourly " + " WHERE space_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (child_space_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_child_space_hourly[child_space_id] = None + else: + energy_child_space_hourly[child_space_id] = dict() + for row_energy_value in rows_energy_values: + current_datetime_utc = row_energy_value[0] + if current_datetime_utc not in energy_child_space_hourly[child_space_id]: + energy_child_space_hourly[child_space_id][current_datetime_utc] = dict() + energy_category_id = row_energy_value[1] + actual_value = row_energy_value[2] + energy_child_space_hourly[child_space_id][current_datetime_utc][energy_category_id] = actual_value + except Exception as e: + error_string = "Error in step 7 of space_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 8: determine common time slot to aggregate + #################################################################################################################### + + common_start_datetime_utc = start_datetime_utc + common_end_datetime_utc = end_datetime_utc + + print("Getting common time slot of energy values for all combined equipments") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_combined_equipment_hourly is not None and len(energy_combined_equipment_hourly) > 0: + for combined_equipment_id, energy_hourly in energy_combined_equipment_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all equipments...") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_equipment_hourly is not None and len(energy_equipment_hourly) > 0: + for equipment_id, energy_hourly in energy_equipment_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all child spaces...") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_child_space_hourly is not None and len(energy_child_space_hourly) > 0: + for child_space_id, energy_hourly in energy_child_space_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + if (energy_combined_equipment_hourly is None or len(energy_combined_equipment_hourly) == 0) and \ + (energy_equipment_hourly is None or len(energy_equipment_hourly) == 0) and \ + (energy_child_space_hourly is None or len(energy_child_space_hourly) == 0): + # There isn't any energy data + print("There isn't any energy data") + # continue the for space loop to the next space + print("continue the for space loop to the next space") + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + return None + + print("common_start_datetime_utc: " + str(common_start_datetime_utc)) + print("common_end_datetime_utc: " + str(common_end_datetime_utc)) + + #################################################################################################################### + # Step 9: aggregate energy data in the common time slot by energy categories and hourly + #################################################################################################################### + + print("Step 9: aggregate energy data in the common time slot by energy categories and hourly") + aggregated_values = list() + try: + current_datetime_utc = common_start_datetime_utc + while common_start_datetime_utc is not None \ + and common_end_datetime_utc is not None \ + and current_datetime_utc <= common_end_datetime_utc: + aggregated_value = dict() + aggregated_value['start_datetime_utc'] = current_datetime_utc + aggregated_value['meta_data'] = dict() + + if combined_equipment_list is not None and len(combined_equipment_list) > 0: + for combined_equipment in combined_equipment_list: + combined_equipment_id = str(combined_equipment['id']) + meta_data_dict = \ + energy_combined_equipment_hourly[combined_equipment_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_category_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if equipment_list is not None and len(equipment_list) > 0: + for equipment in equipment_list: + equipment_id = str(equipment['id']) + meta_data_dict = energy_equipment_hourly[equipment_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_category_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if child_space_list is not None and len(child_space_list) > 0: + for child_space in child_space_list: + child_space_id = str(child_space['id']) + meta_data_dict = energy_child_space_hourly[child_space_id].get(current_datetime_utc, None) + if meta_data_dict is not None and len(meta_data_dict) > 0: + for energy_category_id, actual_value in meta_data_dict.items(): + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + aggregated_values.append(aggregated_value) + + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + except Exception as e: + error_string = "Error in step 9 of space_energy_output_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 10: save energy data to energy database + #################################################################################################################### + print("Step 10: save energy data to energy database") + + if len(aggregated_values) > 0: + try: + add_values = (" INSERT INTO tbl_space_output_category_hourly " + " (space_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for aggregated_value in aggregated_values: + for energy_category_id, actual_value in aggregated_value['meta_data'].items(): + add_values += " (" + str(space['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "'," + add_values += str(actual_value) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_energy_db.execute(add_values[:-2]) + cnx_energy_db.commit() + + except Exception as e: + error_string = "Error in step 8 of space_energy_output_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + else: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() diff --git a/myems-aggregation/store_billing_input_category.py b/myems-aggregation/store_billing_input_category.py new file mode 100644 index 00000000..6813921c --- /dev/null +++ b/myems-aggregation/store_billing_input_category.py @@ -0,0 +1,269 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +import tariff +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all stores +# for each store in list: +# Step 2: get the latest start_datetime_utc +# Step 3: get all energy input data since the latest start_datetime_utc +# Step 4: get tariffs +# Step 5: calculate billing by multiplying energy with tariff +# Step 6: save billing data to database +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all stores + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of store_billing_input_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name, cost_center_id " + " FROM tbl_stores " + " ORDER BY id ") + rows_stores = cursor_system_db.fetchall() + + if rows_stores is None or len(rows_stores) == 0: + print("Step 1.2: There isn't any stores. ") + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + store_list = list() + for row in rows_stores: + store_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]}) + + except Exception as e: + logger.error("Error in step 1.2 of store_billing_input_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Step 1.2: Got all stores from MyEMS System Database") + + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + logger.error("Error in step 1.3 of store_billing_input_category " + str(e)) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Energy Database") + + cnx_billing_db = None + cursor_billing_db = None + try: + cnx_billing_db = mysql.connector.connect(**config.myems_billing_db) + cursor_billing_db = cnx_billing_db.cursor() + except Exception as e: + logger.error("Error in step 1.4 of store_billing_input_category " + str(e)) + if cursor_billing_db: + cursor_billing_db.close() + if cnx_billing_db: + cnx_billing_db.close() + + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Billing Database") + + for store in store_list: + + ############################################################################################################ + # Step 2: get the latest start_datetime_utc + ############################################################################################################ + print("Step 2: get the latest start_datetime_utc from billing database for " + store['name']) + try: + cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) " + " FROM tbl_store_input_category_hourly " + " WHERE store_id = %s ", + (store['id'], )) + row_datetime = cursor_billing_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]) + except Exception as e: + logger.error("Error in step 2 of store_billing_input_category " + str(e)) + # break the for store loop + break + + ############################################################################################################ + # Step 3: get all energy input data since the latest start_datetime_utc + ############################################################################################################ + print("Step 3: get all energy input data since the latest start_datetime_utc") + + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_store_input_category_hourly " + " WHERE store_id = %s AND start_datetime_utc >= %s " + " ORDER BY id ") + cursor_energy_db.execute(query, (store['id'], start_datetime_utc, )) + rows_hourly = cursor_energy_db.fetchall() + + if rows_hourly is None or len(rows_hourly) == 0: + print("Step 3: There isn't any energy input data to calculate. ") + # continue the for store loop + continue + + energy_dict = dict() + energy_category_list = list() + end_datetime_utc = start_datetime_utc + for row_hourly in rows_hourly: + current_datetime_utc = row_hourly[0] + energy_category_id = row_hourly[1] + + if energy_category_id not in energy_category_list: + energy_category_list.append(energy_category_id) + + actual_value = row_hourly[2] + if energy_dict.get(current_datetime_utc) is None: + energy_dict[current_datetime_utc] = dict() + energy_dict[current_datetime_utc][energy_category_id] = actual_value + if current_datetime_utc > end_datetime_utc: + end_datetime_utc = current_datetime_utc + + ############################################################################################################ + # Step 4: get tariffs + ############################################################################################################ + print("Step 4: get tariffs") + tariff_dict = dict() + for energy_category_id in energy_category_list: + tariff_dict[energy_category_id] = tariff.get_energy_category_tariffs(store['cost_center_id'], + energy_category_id, + start_datetime_utc, + end_datetime_utc) + ############################################################################################################ + # Step 5: calculate billing by multiplying energy with tariff + ############################################################################################################ + print("Step 5: calculate billing by multiplying energy with tariff") + billing_dict = dict() + + if len(energy_dict) > 0: + for current_datetime_utc in energy_dict.keys(): + billing_dict[current_datetime_utc] = dict() + for energy_category_id in energy_category_list: + current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc) + current_energy = energy_dict[current_datetime_utc].get(energy_category_id) + if current_tariff is not None \ + and isinstance(current_tariff, Decimal) \ + and current_energy is not None \ + and isinstance(current_energy, Decimal): + billing_dict[current_datetime_utc][energy_category_id] = \ + current_energy * current_tariff + + if len(billing_dict[current_datetime_utc]) == 0: + del billing_dict[current_datetime_utc] + + ############################################################################################################ + # Step 6: save billing data to billing database + ############################################################################################################ + print("Step 6: save billing data to billing database") + + if len(billing_dict) > 0: + try: + add_values = (" INSERT INTO tbl_store_input_category_hourly " + " (store_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for current_datetime_utc in billing_dict: + for energy_category_id in energy_category_list: + current_billing = billing_dict[current_datetime_utc].get(energy_category_id) + if current_billing is not None and isinstance(current_billing, Decimal): + add_values += " (" + str(store['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + current_datetime_utc.isoformat()[0:19] + "'," + add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_billing_db.execute(add_values[:-2]) + cnx_billing_db.commit() + except Exception as e: + logger.error("Error in step 6 of store_billing_input_category " + str(e)) + # break the for store loop + break + + # end of for store loop + if cnx_system_db: + cnx_system_db.close() + if cursor_system_db: + cursor_system_db.close() + + if cnx_energy_db: + cnx_energy_db.close() + if cursor_energy_db: + cursor_energy_db.close() + + if cnx_billing_db: + cnx_billing_db.close() + if cursor_billing_db: + cursor_billing_db.close() + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of the outermost while loop diff --git a/myems-aggregation/store_billing_input_item.py b/myems-aggregation/store_billing_input_item.py new file mode 100644 index 00000000..37787059 --- /dev/null +++ b/myems-aggregation/store_billing_input_item.py @@ -0,0 +1,269 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +import tariff +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all stores +# for each store in list: +# Step 2: get the latest start_datetime_utc +# Step 3: get all energy input data since the latest start_datetime_utc +# Step 4: get tariffs +# Step 5: calculate billing by multiplying energy with tariff +# Step 6: save billing data to database +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all stores + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of store_billing_input_item " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name, cost_center_id " + " FROM tbl_stores " + " ORDER BY id ") + rows_stores = cursor_system_db.fetchall() + + if rows_stores is None or len(rows_stores) == 0: + print("Step 1.2: There isn't any stores. ") + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + store_list = list() + for row in rows_stores: + store_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]}) + + except Exception as e: + logger.error("Error in step 1.2 of store_billing_input_item " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Step 1.2: Got all stores from MyEMS System Database") + + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + logger.error("Error in step 1.3 of store_billing_input_item " + str(e)) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Energy Database") + + cnx_billing_db = None + cursor_billing_db = None + try: + cnx_billing_db = mysql.connector.connect(**config.myems_billing_db) + cursor_billing_db = cnx_billing_db.cursor() + except Exception as e: + logger.error("Error in step 1.4 of store_billing_input_item " + str(e)) + if cursor_billing_db: + cursor_billing_db.close() + if cnx_billing_db: + cnx_billing_db.close() + + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Billing Database") + + for store in store_list: + + ############################################################################################################ + # Step 2: get the latest start_datetime_utc + ############################################################################################################ + print("Step 2: get the latest start_datetime_utc from billing database for " + store['name']) + try: + cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) " + " FROM tbl_store_input_item_hourly " + " WHERE store_id = %s ", + (store['id'], )) + row_datetime = cursor_billing_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]) + except Exception as e: + logger.error("Error in step 2 of store_billing_input_item " + str(e)) + # break the for store loop + break + + ############################################################################################################ + # Step 3: get all energy input data since the latest start_datetime_utc + ############################################################################################################ + print("Step 3: get all energy input data since the latest start_datetime_utc") + + query = (" SELECT start_datetime_utc, energy_item_id, actual_value " + " FROM tbl_store_input_item_hourly " + " WHERE store_id = %s AND start_datetime_utc >= %s " + " ORDER BY id ") + cursor_energy_db.execute(query, (store['id'], start_datetime_utc, )) + rows_hourly = cursor_energy_db.fetchall() + + if rows_hourly is None or len(rows_hourly) == 0: + print("Step 3: There isn't any energy input data to calculate. ") + # continue the for store loop + continue + + energy_dict = dict() + energy_item_list = list() + end_datetime_utc = start_datetime_utc + for row_hourly in rows_hourly: + current_datetime_utc = row_hourly[0] + energy_item_id = row_hourly[1] + + if energy_item_id not in energy_item_list: + energy_item_list.append(energy_item_id) + + actual_value = row_hourly[2] + if energy_dict.get(current_datetime_utc) is None: + energy_dict[current_datetime_utc] = dict() + energy_dict[current_datetime_utc][energy_item_id] = actual_value + if current_datetime_utc > end_datetime_utc: + end_datetime_utc = current_datetime_utc + + ############################################################################################################ + # Step 4: get tariffs + ############################################################################################################ + print("Step 4: get tariffs") + tariff_dict = dict() + for energy_item_id in energy_item_list: + tariff_dict[energy_item_id] = tariff.get_energy_item_tariffs(store['cost_center_id'], + energy_item_id, + start_datetime_utc, + end_datetime_utc) + ############################################################################################################ + # Step 5: calculate billing by multiplying energy with tariff + ############################################################################################################ + print("Step 5: calculate billing by multiplying energy with tariff") + billing_dict = dict() + + if len(energy_dict) > 0: + for current_datetime_utc in energy_dict.keys(): + billing_dict[current_datetime_utc] = dict() + for energy_item_id in energy_item_list: + current_tariff = tariff_dict[energy_item_id].get(current_datetime_utc) + current_energy = energy_dict[current_datetime_utc].get(energy_item_id) + if current_tariff is not None \ + and isinstance(current_tariff, Decimal) \ + and current_energy is not None \ + and isinstance(current_energy, Decimal): + billing_dict[current_datetime_utc][energy_item_id] = \ + current_energy * current_tariff + + if len(billing_dict[current_datetime_utc]) == 0: + del billing_dict[current_datetime_utc] + + ############################################################################################################ + # Step 6: save billing data to billing database + ############################################################################################################ + print("Step 6: save billing data to billing database") + + if len(billing_dict) > 0: + try: + add_values = (" INSERT INTO tbl_store_input_item_hourly " + " (store_id, " + " energy_item_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for current_datetime_utc in billing_dict: + for energy_item_id in energy_item_list: + current_billing = billing_dict[current_datetime_utc].get(energy_item_id) + if current_billing is not None and isinstance(current_billing, Decimal): + add_values += " (" + str(store['id']) + "," + add_values += " " + str(energy_item_id) + "," + add_values += "'" + current_datetime_utc.isoformat()[0:19] + "'," + add_values += str(billing_dict[current_datetime_utc][energy_item_id]) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_billing_db.execute(add_values[:-2]) + cnx_billing_db.commit() + except Exception as e: + logger.error("Error in step 6 of store_billing_input_item " + str(e)) + # break the for store loop + break + + # end of for store loop + if cnx_system_db: + cnx_system_db.close() + if cursor_system_db: + cursor_system_db.close() + + if cnx_energy_db: + cnx_energy_db.close() + if cursor_energy_db: + cursor_energy_db.close() + + if cnx_billing_db: + cnx_billing_db.close() + if cursor_billing_db: + cursor_billing_db.close() + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of the outermost while loop diff --git a/myems-aggregation/store_energy_input_category.py b/myems-aggregation/store_energy_input_category.py new file mode 100644 index 00000000..dda181db --- /dev/null +++ b/myems-aggregation/store_energy_input_category.py @@ -0,0 +1,521 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +from multiprocessing import Pool +import random +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all stores +# Step 2: Create multiprocessing pool to call worker in parallel +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all stores + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of store_energy_input_category.main " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_stores " + " ORDER BY id ") + rows_stores = cursor_system_db.fetchall() + + if rows_stores is None or len(rows_stores) == 0: + print("There isn't any stores ") + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + + store_list = list() + for row in rows_stores: + store_list.append({"id": row[0], "name": row[1]}) + + except Exception as e: + logger.error("Error in step 1.2 of store_energy_input_category.main " + str(e)) + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + print("Got all stores in MyEMS System Database") + + # shuffle the store list for randomly calculating the meter hourly value + random.shuffle(store_list) + + ################################################################################################################ + # Step 2: Create multiprocessing pool to call worker in parallel + ################################################################################################################ + p = Pool(processes=config.pool_size) + error_list = p.map(worker, store_list) + p.close() + p.join() + + for error in error_list: + if error is not None and len(error) > 0: + logger.error(error) + + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of outer while + + +######################################################################################################################## +# PROCEDURES: +# Step 1: get all input meters associated with the store +# Step 2: get all input virtual meters associated with the store +# Step 3: get all input offline meters associated with the store +# Step 4: determine start datetime and end datetime to aggregate +# Step 5: for each meter in list, get energy input data from energy database +# Step 6: for each virtual meter in list, get energy input data from energy database +# Step 7: for each offline meter in list, get energy input data from energy database +# Step 8: determine common time slot to aggregate +# Step 9: aggregate energy data in the common time slot by energy categories and hourly +# Step 10: save energy data to energy database +# +# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter +######################################################################################################################## + +def worker(store): + #################################################################################################################### + # Step 1: get all input meters associated with the store + #################################################################################################################### + print("Step 1: get all input meters associated with the store " + str(store['name'])) + + meter_list = list() + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + error_string = "Error in step 1.1 of store_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_meters m, tbl_stores_meters sm " + " WHERE m.id = sm.meter_id " + " AND m.is_counted = true " + " AND sm.store_id = %s ", + (store['id'],)) + rows_meters = cursor_system_db.fetchall() + + if rows_meters is not None and len(rows_meters) > 0: + for row in rows_meters: + meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 1.2 of store_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 2: get all input virtual meters associated with the store + #################################################################################################################### + print("Step 2: get all input virtual meters associated with the store") + virtual_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_virtual_meters m, tbl_stores_virtual_meters sm " + " WHERE m.id = sm.virtual_meter_id " + " AND m.is_counted = true " + " AND sm.store_id = %s ", + (store['id'],)) + rows_virtual_meters = cursor_system_db.fetchall() + + if rows_virtual_meters is not None and len(rows_virtual_meters) > 0: + for row in rows_virtual_meters: + virtual_meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 2.1 of store_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 3: get all input offline meters associated with the store + #################################################################################################################### + print("Step 3: get all input offline meters associated with the store") + + offline_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_offline_meters m, tbl_stores_offline_meters sm " + " WHERE m.id = sm.offline_meter_id " + " AND m.is_counted = true " + " AND sm.store_id = %s ", + (store['id'],)) + rows_offline_meters = cursor_system_db.fetchall() + + if rows_offline_meters is not None and len(rows_offline_meters) > 0: + for row in rows_offline_meters: + offline_meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 3.1 of store_energy_input_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + #################################################################################################################### + # stop to the next store if this store is empty + #################################################################################################################### + if (meter_list is None or len(meter_list) == 0) and \ + (virtual_meter_list is None or len(virtual_meter_list) == 0) and \ + (offline_meter_list is None or len(offline_meter_list) == 0): + print("This is an empty store ") + return None + + #################################################################################################################### + # Step 4: determine start datetime and end datetime to aggregate + #################################################################################################################### + print("Step 4: determine start datetime and end datetime to aggregate") + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + error_string = "Error in step 4.1 of store_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + try: + query = (" SELECT MAX(start_datetime_utc) " + " FROM tbl_store_input_category_hourly " + " WHERE store_id = %s ") + cursor_energy_db.execute(query, (store['id'],)) + row_datetime = cursor_energy_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19] + + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19]) + + except Exception as e: + error_string = "Error in step 4.2 of store_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 5: for each meter in list, get energy input data from energy database + #################################################################################################################### + energy_meter_hourly = dict() + try: + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_meter_hourly " + " WHERE meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_meter_hourly[meter_id] = None + else: + energy_meter_hourly[meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 5.1 of store_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 6: for each virtual meter in list, get energy input data from energy database + #################################################################################################################### + energy_virtual_meter_hourly = dict() + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + try: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_virtual_meter_hourly " + " WHERE virtual_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_virtual_meter_hourly[virtual_meter_id] = None + else: + energy_virtual_meter_hourly[virtual_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 6.1 of store_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 7: for each offline meter in list, get energy input data from energy database + #################################################################################################################### + energy_offline_meter_hourly = dict() + if offline_meter_list is not None and len(offline_meter_list) > 0: + try: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_offline_meter_hourly " + " WHERE offline_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_offline_meter_hourly[offline_meter_id] = None + else: + energy_offline_meter_hourly[offline_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1] + + except Exception as e: + error_string = "Error in step 7.1 of store_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 8: determine common time slot to aggregate + #################################################################################################################### + + common_start_datetime_utc = start_datetime_utc + common_end_datetime_utc = end_datetime_utc + + print("Getting common time slot of energy values for all meters") + if energy_meter_hourly is not None and len(energy_meter_hourly) > 0: + for meter_id, energy_hourly in energy_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all virtual meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0: + for meter_id, energy_hourly in energy_virtual_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all offline meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0: + for meter_id, energy_hourly in energy_offline_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \ + (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \ + (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0): + # There isn't any energy data + print("There isn't any energy data") + # continue the for store loop to the next store + print("continue the for store loop to the next store") + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + return None + + print("common_start_datetime_utc: " + str(common_start_datetime_utc)) + print("common_end_datetime_utc: " + str(common_end_datetime_utc)) + + #################################################################################################################### + # Step 9: aggregate energy data in the common time slot by energy categories and hourly + #################################################################################################################### + + print("Step 9: aggregate energy data in the common time slot by energy categories and hourly") + aggregated_values = list() + try: + current_datetime_utc = common_start_datetime_utc + while common_start_datetime_utc is not None \ + and common_end_datetime_utc is not None \ + and current_datetime_utc <= common_end_datetime_utc: + aggregated_value = dict() + aggregated_value['start_datetime_utc'] = current_datetime_utc + aggregated_value['meta_data'] = dict() + + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + energy_category_id = meter['energy_category_id'] + actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + energy_category_id = virtual_meter['energy_category_id'] + actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if offline_meter_list is not None and len(offline_meter_list) > 0: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + energy_category_id = offline_meter['energy_category_id'] + actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + aggregated_values.append(aggregated_value) + + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + except Exception as e: + error_string = "Error in step 9 of store_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 10: save energy data to energy database + #################################################################################################################### + print("Step 10: save energy data to energy database") + + if len(aggregated_values) > 0: + try: + add_values = (" INSERT INTO tbl_store_input_category_hourly " + " (store_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for aggregated_value in aggregated_values: + for energy_category_id, actual_value in aggregated_value['meta_data'].items(): + add_values += " (" + str(store['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "'," + add_values += str(actual_value) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_energy_db.execute(add_values[:-2]) + cnx_energy_db.commit() + + except Exception as e: + error_string = "Error in step 10.1 of store_energy_input_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + else: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() diff --git a/myems-aggregation/store_energy_input_item.py b/myems-aggregation/store_energy_input_item.py new file mode 100644 index 00000000..b9c71925 --- /dev/null +++ b/myems-aggregation/store_energy_input_item.py @@ -0,0 +1,524 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +from multiprocessing import Pool +import random +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all stores +# Step 2: Create multiprocessing pool to call worker in parallel +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all stores + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of store_energy_input_item.main " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_stores " + " ORDER BY id ") + rows_stores = cursor_system_db.fetchall() + + if rows_stores is None or len(rows_stores) == 0: + print("There isn't any stores ") + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + + store_list = list() + for row in rows_stores: + store_list.append({"id": row[0], "name": row[1]}) + + except Exception as e: + logger.error("Error in step 1.2 of store_energy_input_item.main " + str(e)) + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + print("Got all stores in MyEMS System Database") + + # shuffle the store list for randomly calculating the meter hourly value + random.shuffle(store_list) + + ################################################################################################################ + # Step 2: Create multiprocessing pool to call worker in parallel + ################################################################################################################ + p = Pool(processes=config.pool_size) + error_list = p.map(worker, store_list) + p.close() + p.join() + + for error in error_list: + if error is not None and len(error) > 0: + logger.error(error) + + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of outer while + + +######################################################################################################################## +# PROCEDURES: +# Step 1: get all input meters associated with the store +# Step 2: get all input virtual meters associated with the store +# Step 3: get all input offline meters associated with the store +# Step 4: determine start datetime and end datetime to aggregate +# Step 5: for each meter in list, get energy input data from energy database +# Step 6: for each virtual meter in list, get energy input data from energy database +# Step 7: for each offline meter in list, get energy input data from energy database +# Step 8: determine common time slot to aggregate +# Step 9: aggregate energy data in the common time slot by energy items and hourly +# Step 10: save energy data to energy database +# +# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter +######################################################################################################################## + +def worker(store): + #################################################################################################################### + # Step 1: get all input meters associated with the store + #################################################################################################################### + print("Step 1: get all input meters associated with the store " + str(store['name'])) + + meter_list = list() + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + error_string = "Error in step 1.1 of store_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_meters m, tbl_stores_meters sm " + " WHERE m.id = sm.meter_id " + " AND m.is_counted = true " + " AND m.energy_item_id is NOT NULL " + " AND sm.store_id = %s ", + (store['id'],)) + rows_meters = cursor_system_db.fetchall() + + if rows_meters is not None and len(rows_meters) > 0: + for row in rows_meters: + meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 1.2 of store_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 2: get all input virtual meters associated with the store + #################################################################################################################### + print("Step 2: get all input virtual meters associated with the store") + virtual_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_virtual_meters m, tbl_stores_virtual_meters sm " + " WHERE m.id = sm.virtual_meter_id " + " AND m.energy_item_id is NOT NULL " + " AND m.is_counted = true " + " AND sm.store_id = %s ", + (store['id'],)) + rows_virtual_meters = cursor_system_db.fetchall() + + if rows_virtual_meters is not None and len(rows_virtual_meters) > 0: + for row in rows_virtual_meters: + virtual_meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 2.1 of store_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 3: get all input offline meters associated with the store + #################################################################################################################### + print("Step 3: get all input offline meters associated with the store") + + offline_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_offline_meters m, tbl_stores_offline_meters sm " + " WHERE m.id = sm.offline_meter_id " + " AND m.energy_item_id is NOT NULL " + " AND m.is_counted = true " + " AND sm.store_id = %s ", + (store['id'],)) + rows_offline_meters = cursor_system_db.fetchall() + + if rows_offline_meters is not None and len(rows_offline_meters) > 0: + for row in rows_offline_meters: + offline_meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 3.1 of store_energy_input_item.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + #################################################################################################################### + # stop to the next store if this store is empty + #################################################################################################################### + if (meter_list is None or len(meter_list) == 0) and \ + (virtual_meter_list is None or len(virtual_meter_list) == 0) and \ + (offline_meter_list is None or len(offline_meter_list) == 0): + print("This is an empty store ") + return None + + #################################################################################################################### + # Step 4: determine start datetime and end datetime to aggregate + #################################################################################################################### + print("Step 4: determine start datetime and end datetime to aggregate") + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + error_string = "Error in step 4.1 of store_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + try: + query = (" SELECT MAX(start_datetime_utc) " + " FROM tbl_store_input_item_hourly " + " WHERE store_id = %s ") + cursor_energy_db.execute(query, (store['id'],)) + row_datetime = cursor_energy_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19] + + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19]) + + except Exception as e: + error_string = "Error in step 4.2 of store_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 5: for each meter in list, get energy input data from energy database + #################################################################################################################### + energy_meter_hourly = dict() + try: + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_meter_hourly " + " WHERE meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_meter_hourly[meter_id] = None + else: + energy_meter_hourly[meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 5.1 of store_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 6: for each virtual meter in list, get energy input data from energy database + #################################################################################################################### + energy_virtual_meter_hourly = dict() + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + try: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_virtual_meter_hourly " + " WHERE virtual_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_virtual_meter_hourly[virtual_meter_id] = None + else: + energy_virtual_meter_hourly[virtual_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 6.1 of store_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 7: for each offline meter in list, get energy input data from energy database + #################################################################################################################### + energy_offline_meter_hourly = dict() + if offline_meter_list is not None and len(offline_meter_list) > 0: + try: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_offline_meter_hourly " + " WHERE offline_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_offline_meter_hourly[offline_meter_id] = None + else: + energy_offline_meter_hourly[offline_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1] + + except Exception as e: + error_string = "Error in step 7.1 of store_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 8: determine common time slot to aggregate + #################################################################################################################### + + common_start_datetime_utc = start_datetime_utc + common_end_datetime_utc = end_datetime_utc + + print("Getting common time slot of energy values for all meters") + if energy_meter_hourly is not None and len(energy_meter_hourly) > 0: + for meter_id, energy_hourly in energy_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all virtual meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0: + for meter_id, energy_hourly in energy_virtual_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all offline meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0: + for meter_id, energy_hourly in energy_offline_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \ + (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \ + (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0): + # There isn't any energy data + print("There isn't any energy data") + # continue the for store loop to the next store + print("continue the for store loop to the next store") + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + return None + + print("common_start_datetime_utc: " + str(common_start_datetime_utc)) + print("common_end_datetime_utc: " + str(common_end_datetime_utc)) + + #################################################################################################################### + # Step 9: aggregate energy data in the common time slot by energy items and hourly + #################################################################################################################### + + print("Step 9: aggregate energy data in the common time slot by energy items and hourly") + aggregated_values = list() + try: + current_datetime_utc = common_start_datetime_utc + while common_start_datetime_utc is not None \ + and common_end_datetime_utc is not None \ + and current_datetime_utc <= common_end_datetime_utc: + aggregated_value = dict() + aggregated_value['start_datetime_utc'] = current_datetime_utc + aggregated_value['meta_data'] = dict() + + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + energy_item_id = meter['energy_item_id'] + actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + energy_item_id = virtual_meter['energy_item_id'] + actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if offline_meter_list is not None and len(offline_meter_list) > 0: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + energy_item_id = offline_meter['energy_item_id'] + actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + aggregated_values.append(aggregated_value) + + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + except Exception as e: + error_string = "Error in step 9 of store_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 10: save energy data to energy database + #################################################################################################################### + print("Step 10: save energy data to energy database") + + if len(aggregated_values) > 0: + try: + add_values = (" INSERT INTO tbl_store_input_item_hourly " + " (store_id, " + " energy_item_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for aggregated_value in aggregated_values: + for energy_item_id, actual_value in aggregated_value['meta_data'].items(): + add_values += " (" + str(store['id']) + "," + add_values += " " + str(energy_item_id) + "," + add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "'," + add_values += str(actual_value) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_energy_db.execute(add_values[:-2]) + cnx_energy_db.commit() + + except Exception as e: + error_string = "Error in step 10.1 of store_energy_input_item.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + else: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() diff --git a/myems-aggregation/tariff.py b/myems-aggregation/tariff.py new file mode 100644 index 00000000..ca76cf3c --- /dev/null +++ b/myems-aggregation/tariff.py @@ -0,0 +1,205 @@ +from datetime import timedelta +import mysql.connector +import config +import collections + + +######################################################################################################################## +# Get tariffs by energy category +######################################################################################################################## +def get_energy_category_tariffs(cost_center_id, energy_category_id, start_datetime_utc, end_datetime_utc): + # todo: verify parameters + if cost_center_id is None: + return dict() + + # get timezone offset in minutes, this value will be returned to client + timezone_offset = int(config.utc_offset[1:3]) * 60 + int(config.utc_offset[4:6]) + if config.utc_offset[0] == '-': + timezone_offset = -timezone_offset + + tariff_dict = collections.OrderedDict() + + cnx = None + cursor = None + try: + cnx = mysql.connector.connect(**config.myems_system_db) + cursor = cnx.cursor() + query_tariffs = (" SELECT t.id, t.valid_from_datetime_utc, t.valid_through_datetime_utc " + " FROM tbl_tariffs t, tbl_cost_centers_tariffs cct " + " WHERE t.energy_category_id = %s AND " + " t.id = cct.tariff_id AND " + " cct.cost_center_id = %s AND " + " t.valid_through_datetime_utc >= %s AND " + " t.valid_from_datetime_utc <= %s " + " ORDER BY t.valid_from_datetime_utc ") + cursor.execute(query_tariffs, (energy_category_id, cost_center_id, start_datetime_utc, end_datetime_utc,)) + rows_tariffs = cursor.fetchall() + except Exception as e: + print(str(e)) + if cnx: + cnx.disconnect() + if cursor: + cursor.close() + return dict() + + if rows_tariffs is None or len(rows_tariffs) == 0: + if cursor: + cursor.close() + if cnx: + cnx.disconnect() + return dict() + + for row in rows_tariffs: + tariff_dict[row[0]] = {'valid_from_datetime_utc': row[1], + 'valid_through_datetime_utc': row[2], + 'rates': list()} + + try: + query_timeofuse_tariffs = (" SELECT tariff_id, start_time_of_day, end_time_of_day, price " + " FROM tbl_tariffs_timeofuses " + " WHERE tariff_id IN ( " + ', '.join(map(str, tariff_dict.keys())) + ")" + " ORDER BY tariff_id, start_time_of_day ") + cursor.execute(query_timeofuse_tariffs, ) + rows_timeofuse_tariffs = cursor.fetchall() + except Exception as e: + print(str(e)) + if cnx: + cnx.disconnect() + if cursor: + cursor.close() + return dict() + + if cursor: + cursor.close() + if cnx: + cnx.disconnect() + + if rows_timeofuse_tariffs is None or len(rows_timeofuse_tariffs) == 0: + return dict() + + for row in rows_timeofuse_tariffs: + tariff_dict[row[0]]['rates'].append({'start_time_of_day': row[1], + 'end_time_of_day': row[2], + 'price': row[3]}) + + result = dict() + for tariff_id, tariff_value in tariff_dict.items(): + current_datetime_utc = tariff_value['valid_from_datetime_utc'] + while current_datetime_utc < tariff_value['valid_through_datetime_utc']: + for rate in tariff_value['rates']: + current_datetime_local = current_datetime_utc + timedelta(minutes=timezone_offset) + seconds_since_midnight = (current_datetime_local - + current_datetime_local.replace(hour=0, + second=0, + microsecond=0, + tzinfo=None)).total_seconds() + if rate['start_time_of_day'].total_seconds() <= \ + seconds_since_midnight < rate['end_time_of_day'].total_seconds(): + result[current_datetime_utc] = rate['price'] + break + + # start from the next time slot + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + return {k: v for k, v in result.items() if start_datetime_utc <= k <= end_datetime_utc} + + +######################################################################################################################## +# Get tariffs by energy item +######################################################################################################################## +def get_energy_item_tariffs(cost_center_id, energy_item_id, start_datetime_utc, end_datetime_utc): + # todo: verify parameters + if cost_center_id is None: + return dict() + + # get timezone offset in minutes, this value will be returned to client + timezone_offset = int(config.utc_offset[1:3]) * 60 + int(config.utc_offset[4:6]) + if config.utc_offset[0] == '-': + timezone_offset = -timezone_offset + + tariff_dict = collections.OrderedDict() + + cnx = None + cursor = None + try: + cnx = mysql.connector.connect(**config.myems_system_db) + cursor = cnx.cursor() + query_tariffs = (" SELECT t.id, t.valid_from_datetime_utc, t.valid_through_datetime_utc " + " FROM tbl_tariffs t, tbl_cost_centers_tariffs cct, tbl_energy_items ei " + " WHERE ei.id = %s AND " + " t.energy_category_id = ei.energy_category_id AND " + " t.id = cct.tariff_id AND " + " cct.cost_center_id = %s AND " + " t.valid_through_datetime_utc >= %s AND " + " t.valid_from_datetime_utc <= %s " + " ORDER BY t.valid_from_datetime_utc ") + cursor.execute(query_tariffs, (energy_item_id, cost_center_id, start_datetime_utc, end_datetime_utc,)) + rows_tariffs = cursor.fetchall() + except Exception as e: + print(str(e)) + if cnx: + cnx.disconnect() + if cursor: + cursor.close() + return dict() + + if rows_tariffs is None or len(rows_tariffs) == 0: + if cursor: + cursor.close() + if cnx: + cnx.disconnect() + return dict() + + for row in rows_tariffs: + tariff_dict[row[0]] = {'valid_from_datetime_utc': row[1], + 'valid_through_datetime_utc': row[2], + 'rates': list()} + + try: + query_timeofuse_tariffs = (" SELECT tariff_id, start_time_of_day, end_time_of_day, price " + " FROM tbl_tariffs_timeofuses " + " WHERE tariff_id IN ( " + ', '.join(map(str, tariff_dict.keys())) + ")" + " ORDER BY tariff_id, start_time_of_day ") + cursor.execute(query_timeofuse_tariffs, ) + rows_timeofuse_tariffs = cursor.fetchall() + except Exception as e: + print(str(e)) + if cnx: + cnx.disconnect() + if cursor: + cursor.close() + return dict() + + if cursor: + cursor.close() + if cnx: + cnx.disconnect() + + if rows_timeofuse_tariffs is None or len(rows_timeofuse_tariffs) == 0: + return dict() + + for row in rows_timeofuse_tariffs: + tariff_dict[row[0]]['rates'].append({'start_time_of_day': row[1], + 'end_time_of_day': row[2], + 'price': row[3]}) + + result = dict() + for tariff_id, tariff_value in tariff_dict.items(): + current_datetime_utc = tariff_value['valid_from_datetime_utc'] + while current_datetime_utc < tariff_value['valid_through_datetime_utc']: + for rate in tariff_value['rates']: + current_datetime_local = current_datetime_utc + timedelta(minutes=timezone_offset) + seconds_since_midnight = (current_datetime_local - + current_datetime_local.replace(hour=0, + second=0, + microsecond=0, + tzinfo=None)).total_seconds() + if rate['start_time_of_day'].total_seconds() <= \ + seconds_since_midnight < rate['end_time_of_day'].total_seconds(): + result[current_datetime_utc] = rate['price'] + break + + # start from the next time slot + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + return {k: v for k, v in result.items() if start_datetime_utc <= k <= end_datetime_utc} diff --git a/myems-aggregation/tenant_billing_input_category.py b/myems-aggregation/tenant_billing_input_category.py new file mode 100644 index 00000000..29fff4eb --- /dev/null +++ b/myems-aggregation/tenant_billing_input_category.py @@ -0,0 +1,269 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +import tariff +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all tenants +# for each tenant in list: +# Step 2: get the latest start_datetime_utc +# Step 3: get all energy input data since the latest start_datetime_utc +# Step 4: get tariffs +# Step 5: calculate billing by multiplying energy with tariff +# Step 6: save billing data to database +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all tenants + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of tenant_billing_input_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name, cost_center_id " + " FROM tbl_tenants " + " ORDER BY id ") + rows_tenants = cursor_system_db.fetchall() + + if rows_tenants is None or len(rows_tenants) == 0: + print("Step 1.2: There isn't any tenants. ") + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + tenant_list = list() + for row in rows_tenants: + tenant_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]}) + + except Exception as e: + logger.error("Error in step 1.2 of tenant_billing_input_category " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Step 1.2: Got all tenants from MyEMS System Database") + + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + logger.error("Error in step 1.3 of tenant_billing_input_category " + str(e)) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Energy Database") + + cnx_billing_db = None + cursor_billing_db = None + try: + cnx_billing_db = mysql.connector.connect(**config.myems_billing_db) + cursor_billing_db = cnx_billing_db.cursor() + except Exception as e: + logger.error("Error in step 1.4 of tenant_billing_input_category " + str(e)) + if cursor_billing_db: + cursor_billing_db.close() + if cnx_billing_db: + cnx_billing_db.close() + + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Billing Database") + + for tenant in tenant_list: + + ############################################################################################################ + # Step 2: get the latest start_datetime_utc + ############################################################################################################ + print("Step 2: get the latest start_datetime_utc from billing database for " + tenant['name']) + try: + cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) " + " FROM tbl_tenant_input_category_hourly " + " WHERE tenant_id = %s ", + (tenant['id'], )) + row_datetime = cursor_billing_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]) + except Exception as e: + logger.error("Error in step 2 of tenant_billing_input_category " + str(e)) + # break the for tenant loop + break + + ############################################################################################################ + # Step 3: get all energy input data since the latest start_datetime_utc + ############################################################################################################ + print("Step 3: get all energy input data since the latest start_datetime_utc") + + query = (" SELECT start_datetime_utc, energy_category_id, actual_value " + " FROM tbl_tenant_input_category_hourly " + " WHERE tenant_id = %s AND start_datetime_utc >= %s " + " ORDER BY id ") + cursor_energy_db.execute(query, (tenant['id'], start_datetime_utc, )) + rows_hourly = cursor_energy_db.fetchall() + + if rows_hourly is None or len(rows_hourly) == 0: + print("Step 3: There isn't any energy input data to calculate. ") + # continue the for tenant loop + continue + + energy_dict = dict() + energy_category_list = list() + end_datetime_utc = start_datetime_utc + for row_hourly in rows_hourly: + current_datetime_utc = row_hourly[0] + energy_category_id = row_hourly[1] + + if energy_category_id not in energy_category_list: + energy_category_list.append(energy_category_id) + + actual_value = row_hourly[2] + if energy_dict.get(current_datetime_utc) is None: + energy_dict[current_datetime_utc] = dict() + energy_dict[current_datetime_utc][energy_category_id] = actual_value + if current_datetime_utc > end_datetime_utc: + end_datetime_utc = current_datetime_utc + + ############################################################################################################ + # Step 4: get tariffs + ############################################################################################################ + print("Step 4: get tariffs") + tariff_dict = dict() + for energy_category_id in energy_category_list: + tariff_dict[energy_category_id] = tariff.get_energy_category_tariffs(tenant['cost_center_id'], + energy_category_id, + start_datetime_utc, + end_datetime_utc) + ############################################################################################################ + # Step 5: calculate billing by multiplying energy with tariff + ############################################################################################################ + print("Step 5: calculate billing by multiplying energy with tariff") + billing_dict = dict() + + if len(energy_dict) > 0: + for current_datetime_utc in energy_dict.keys(): + billing_dict[current_datetime_utc] = dict() + for energy_category_id in energy_category_list: + current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc) + current_energy = energy_dict[current_datetime_utc].get(energy_category_id) + if current_tariff is not None \ + and isinstance(current_tariff, Decimal) \ + and current_energy is not None \ + and isinstance(current_energy, Decimal): + billing_dict[current_datetime_utc][energy_category_id] = \ + current_energy * current_tariff + + if len(billing_dict[current_datetime_utc]) == 0: + del billing_dict[current_datetime_utc] + + ############################################################################################################ + # Step 6: save billing data to billing database + ############################################################################################################ + print("Step 6: save billing data to billing database") + + if len(billing_dict) > 0: + try: + add_values = (" INSERT INTO tbl_tenant_input_category_hourly " + " (tenant_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for current_datetime_utc in billing_dict: + for energy_category_id in energy_category_list: + current_billing = billing_dict[current_datetime_utc].get(energy_category_id) + if current_billing is not None and isinstance(current_billing, Decimal): + add_values += " (" + str(tenant['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + current_datetime_utc.isoformat()[0:19] + "'," + add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_billing_db.execute(add_values[:-2]) + cnx_billing_db.commit() + except Exception as e: + logger.error("Error in step 6 of tenant_billing_input_category " + str(e)) + # break the for tenant loop + break + + # end of for tenant loop + if cnx_system_db: + cnx_system_db.close() + if cursor_system_db: + cursor_system_db.close() + + if cnx_energy_db: + cnx_energy_db.close() + if cursor_energy_db: + cursor_energy_db.close() + + if cnx_billing_db: + cnx_billing_db.close() + if cursor_billing_db: + cursor_billing_db.close() + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of the outermost while loop diff --git a/myems-aggregation/tenant_billing_input_item.py b/myems-aggregation/tenant_billing_input_item.py new file mode 100644 index 00000000..2a0685ce --- /dev/null +++ b/myems-aggregation/tenant_billing_input_item.py @@ -0,0 +1,269 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +import tariff +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all tenants +# for each tenant in list: +# Step 2: get the latest start_datetime_utc +# Step 3: get all energy input data since the latest start_datetime_utc +# Step 4: get tariffs +# Step 5: calculate billing by multiplying energy with tariff +# Step 6: save billing data to database +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all tenants + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of tenant_billing_input_item " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name, cost_center_id " + " FROM tbl_tenants " + " ORDER BY id ") + rows_tenants = cursor_system_db.fetchall() + + if rows_tenants is None or len(rows_tenants) == 0: + print("Step 1.2: There isn't any tenants. ") + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + tenant_list = list() + for row in rows_tenants: + tenant_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]}) + + except Exception as e: + logger.error("Error in step 1.2 of tenant_billing_input_item " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Step 1.2: Got all tenants from MyEMS System Database") + + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + logger.error("Error in step 1.3 of tenant_billing_input_item " + str(e)) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Energy Database") + + cnx_billing_db = None + cursor_billing_db = None + try: + cnx_billing_db = mysql.connector.connect(**config.myems_billing_db) + cursor_billing_db = cnx_billing_db.cursor() + except Exception as e: + logger.error("Error in step 1.4 of tenant_billing_input_item " + str(e)) + if cursor_billing_db: + cursor_billing_db.close() + if cnx_billing_db: + cnx_billing_db.close() + + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outermost while loop + time.sleep(60) + continue + + print("Connected to MyEMS Billing Database") + + for tenant in tenant_list: + + ############################################################################################################ + # Step 2: get the latest start_datetime_utc + ############################################################################################################ + print("Step 2: get the latest start_datetime_utc from billing database for " + tenant['name']) + try: + cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) " + " FROM tbl_tenant_input_item_hourly " + " WHERE tenant_id = %s ", + (tenant['id'], )) + row_datetime = cursor_billing_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]) + except Exception as e: + logger.error("Error in step 2 of tenant_billing_input_item " + str(e)) + # break the for tenant loop + break + + ############################################################################################################ + # Step 3: get all energy input data since the latest start_datetime_utc + ############################################################################################################ + print("Step 3: get all energy input data since the latest start_datetime_utc") + + query = (" SELECT start_datetime_utc, energy_item_id, actual_value " + " FROM tbl_tenant_input_item_hourly " + " WHERE tenant_id = %s AND start_datetime_utc >= %s " + " ORDER BY id ") + cursor_energy_db.execute(query, (tenant['id'], start_datetime_utc, )) + rows_hourly = cursor_energy_db.fetchall() + + if rows_hourly is None or len(rows_hourly) == 0: + print("Step 3: There isn't any energy input data to calculate. ") + # continue the for tenant loop + continue + + energy_dict = dict() + energy_item_list = list() + end_datetime_utc = start_datetime_utc + for row_hourly in rows_hourly: + current_datetime_utc = row_hourly[0] + energy_item_id = row_hourly[1] + + if energy_item_id not in energy_item_list: + energy_item_list.append(energy_item_id) + + actual_value = row_hourly[2] + if energy_dict.get(current_datetime_utc) is None: + energy_dict[current_datetime_utc] = dict() + energy_dict[current_datetime_utc][energy_item_id] = actual_value + if current_datetime_utc > end_datetime_utc: + end_datetime_utc = current_datetime_utc + + ############################################################################################################ + # Step 4: get tariffs + ############################################################################################################ + print("Step 4: get tariffs") + tariff_dict = dict() + for energy_item_id in energy_item_list: + tariff_dict[energy_item_id] = tariff.get_energy_item_tariffs(tenant['cost_center_id'], + energy_item_id, + start_datetime_utc, + end_datetime_utc) + ############################################################################################################ + # Step 5: calculate billing by multiplying energy with tariff + ############################################################################################################ + print("Step 5: calculate billing by multiplying energy with tariff") + billing_dict = dict() + + if len(energy_dict) > 0: + for current_datetime_utc in energy_dict.keys(): + billing_dict[current_datetime_utc] = dict() + for energy_item_id in energy_item_list: + current_tariff = tariff_dict[energy_item_id].get(current_datetime_utc) + current_energy = energy_dict[current_datetime_utc].get(energy_item_id) + if current_tariff is not None \ + and isinstance(current_tariff, Decimal) \ + and current_energy is not None \ + and isinstance(current_energy, Decimal): + billing_dict[current_datetime_utc][energy_item_id] = \ + current_energy * current_tariff + + if len(billing_dict[current_datetime_utc]) == 0: + del billing_dict[current_datetime_utc] + + ############################################################################################################ + # Step 6: save billing data to billing database + ############################################################################################################ + print("Step 6: save billing data to billing database") + + if len(billing_dict) > 0: + try: + add_values = (" INSERT INTO tbl_tenant_input_item_hourly " + " (tenant_id, " + " energy_item_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for current_datetime_utc in billing_dict: + for energy_item_id in energy_item_list: + current_billing = billing_dict[current_datetime_utc].get(energy_item_id) + if current_billing is not None and isinstance(current_billing, Decimal): + add_values += " (" + str(tenant['id']) + "," + add_values += " " + str(energy_item_id) + "," + add_values += "'" + current_datetime_utc.isoformat()[0:19] + "'," + add_values += str(billing_dict[current_datetime_utc][energy_item_id]) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_billing_db.execute(add_values[:-2]) + cnx_billing_db.commit() + except Exception as e: + logger.error("Error in step 6 of tenant_billing_input_item " + str(e)) + # break the for tenant loop + break + + # end of for tenant loop + if cnx_system_db: + cnx_system_db.close() + if cursor_system_db: + cursor_system_db.close() + + if cnx_energy_db: + cnx_energy_db.close() + if cursor_energy_db: + cursor_energy_db.close() + + if cnx_billing_db: + cnx_billing_db.close() + if cursor_billing_db: + cursor_billing_db.close() + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of the outermost while loop diff --git a/myems-aggregation/tenant_energy_input_category.py b/myems-aggregation/tenant_energy_input_category.py new file mode 100644 index 00000000..5142a281 --- /dev/null +++ b/myems-aggregation/tenant_energy_input_category.py @@ -0,0 +1,521 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +from multiprocessing import Pool +import random +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all tenants +# Step 2: Create multiprocessing pool to call worker in parallel +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all tenants + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of tenant_energy_input_category.main " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_tenants " + " ORDER BY id ") + rows_tenants = cursor_system_db.fetchall() + + if rows_tenants is None or len(rows_tenants) == 0: + print("There isn't any tenants ") + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + + tenant_list = list() + for row in rows_tenants: + tenant_list.append({"id": row[0], "name": row[1]}) + + except Exception as e: + logger.error("Error in step 1.2 of tenant_energy_input_category.main " + str(e)) + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + print("Got all tenants in MyEMS System Database") + + # shuffle the tenant list for randomly calculating the meter hourly value + random.shuffle(tenant_list) + + ################################################################################################################ + # Step 2: Create multiprocessing pool to call worker in parallel + ################################################################################################################ + p = Pool(processes=config.pool_size) + error_list = p.map(worker, tenant_list) + p.close() + p.join() + + for error in error_list: + if error is not None and len(error) > 0: + logger.error(error) + + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of outer while + + +######################################################################################################################## +# PROCEDURES: +# Step 1: get all input meters associated with the tenant +# Step 2: get all input virtual meters associated with the tenant +# Step 3: get all input offline meters associated with the tenant +# Step 4: determine start datetime and end datetime to aggregate +# Step 5: for each meter in list, get energy input data from energy database +# Step 6: for each virtual meter in list, get energy input data from energy database +# Step 7: for each offline meter in list, get energy input data from energy database +# Step 8: determine common time slot to aggregate +# Step 9: aggregate energy data in the common time slot by energy categories and hourly +# Step 10: save energy data to energy database +# +# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter +######################################################################################################################## + +def worker(tenant): + #################################################################################################################### + # Step 1: get all input meters associated with the tenant + #################################################################################################################### + print("Step 1: get all input meters associated with the tenant " + str(tenant['name'])) + + meter_list = list() + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + error_string = "Error in step 1.1 of tenant_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_meters m, tbl_tenants_meters tm " + " WHERE m.id = tm.meter_id " + " AND m.is_counted = true " + " AND tm.tenant_id = %s ", + (tenant['id'],)) + rows_meters = cursor_system_db.fetchall() + + if rows_meters is not None and len(rows_meters) > 0: + for row in rows_meters: + meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 1.2 of tenant_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 2: get all input virtual meters associated with the tenant + #################################################################################################################### + print("Step 2: get all input virtual meters associated with the tenant") + virtual_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_virtual_meters m, tbl_tenants_virtual_meters tm " + " WHERE m.id = tm.virtual_meter_id " + " AND m.is_counted = true " + " AND tm.tenant_id = %s ", + (tenant['id'],)) + rows_virtual_meters = cursor_system_db.fetchall() + + if rows_virtual_meters is not None and len(rows_virtual_meters) > 0: + for row in rows_virtual_meters: + virtual_meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 2.1 of tenant_energy_input_category.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 3: get all input offline meters associated with the tenant + #################################################################################################################### + print("Step 3: get all input offline meters associated with the tenant") + + offline_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id " + " FROM tbl_offline_meters m, tbl_tenants_offline_meters tm " + " WHERE m.id = tm.offline_meter_id " + " AND m.is_counted = true " + " AND tm.tenant_id = %s ", + (tenant['id'],)) + rows_offline_meters = cursor_system_db.fetchall() + + if rows_offline_meters is not None and len(rows_offline_meters) > 0: + for row in rows_offline_meters: + offline_meter_list.append({"id": row[0], + "name": row[1], + "energy_category_id": row[2]}) + + except Exception as e: + error_string = "Error in step 3.1 of tenant_energy_input_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + #################################################################################################################### + # stop to the next tenant if this tenant is empty + #################################################################################################################### + if (meter_list is None or len(meter_list) == 0) and \ + (virtual_meter_list is None or len(virtual_meter_list) == 0) and \ + (offline_meter_list is None or len(offline_meter_list) == 0): + print("This is an empty tenant ") + return None + + #################################################################################################################### + # Step 4: determine start datetime and end datetime to aggregate + #################################################################################################################### + print("Step 4: determine start datetime and end datetime to aggregate") + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + error_string = "Error in step 4.1 of tenant_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + try: + query = (" SELECT MAX(start_datetime_utc) " + " FROM tbl_tenant_input_category_hourly " + " WHERE tenant_id = %s ") + cursor_energy_db.execute(query, (tenant['id'],)) + row_datetime = cursor_energy_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19] + + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19]) + + except Exception as e: + error_string = "Error in step 4.2 of tenant_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 5: for each meter in list, get energy input data from energy database + #################################################################################################################### + energy_meter_hourly = dict() + try: + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_meter_hourly " + " WHERE meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_meter_hourly[meter_id] = None + else: + energy_meter_hourly[meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 5.1 of tenant_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 6: for each virtual meter in list, get energy input data from energy database + #################################################################################################################### + energy_virtual_meter_hourly = dict() + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + try: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_virtual_meter_hourly " + " WHERE virtual_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_virtual_meter_hourly[virtual_meter_id] = None + else: + energy_virtual_meter_hourly[virtual_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 6.1 of tenant_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 7: for each offline meter in list, get energy input data from energy database + #################################################################################################################### + energy_offline_meter_hourly = dict() + if offline_meter_list is not None and len(offline_meter_list) > 0: + try: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_offline_meter_hourly " + " WHERE offline_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_offline_meter_hourly[offline_meter_id] = None + else: + energy_offline_meter_hourly[offline_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1] + + except Exception as e: + error_string = "Error in step 7.1 of tenant_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 8: determine common time slot to aggregate + #################################################################################################################### + + common_start_datetime_utc = start_datetime_utc + common_end_datetime_utc = end_datetime_utc + + print("Getting common time slot of energy values for all meters") + if energy_meter_hourly is not None and len(energy_meter_hourly) > 0: + for meter_id, energy_hourly in energy_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all virtual meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0: + for meter_id, energy_hourly in energy_virtual_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all offline meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0: + for meter_id, energy_hourly in energy_offline_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \ + (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \ + (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0): + # There isn't any energy data + print("There isn't any energy data") + # continue the for tenant loop to the next tenant + print("continue the for tenant loop to the next tenant") + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + return None + + print("common_start_datetime_utc: " + str(common_start_datetime_utc)) + print("common_end_datetime_utc: " + str(common_end_datetime_utc)) + + #################################################################################################################### + # Step 9: aggregate energy data in the common time slot by energy categories and hourly + #################################################################################################################### + + print("Step 9: aggregate energy data in the common time slot by energy categories and hourly") + aggregated_values = list() + try: + current_datetime_utc = common_start_datetime_utc + while common_start_datetime_utc is not None \ + and common_end_datetime_utc is not None \ + and current_datetime_utc <= common_end_datetime_utc: + aggregated_value = dict() + aggregated_value['start_datetime_utc'] = current_datetime_utc + aggregated_value['meta_data'] = dict() + + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + energy_category_id = meter['energy_category_id'] + actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + energy_category_id = virtual_meter['energy_category_id'] + actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + if offline_meter_list is not None and len(offline_meter_list) > 0: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + energy_category_id = offline_meter['energy_category_id'] + actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_category_id] = \ + aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value + + aggregated_values.append(aggregated_value) + + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + except Exception as e: + error_string = "Error in step 9 of tenant_energy_input_category.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 10: save energy data to energy database + #################################################################################################################### + print("Step 10: save energy data to energy database") + + if len(aggregated_values) > 0: + try: + add_values = (" INSERT INTO tbl_tenant_input_category_hourly " + " (tenant_id, " + " energy_category_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for aggregated_value in aggregated_values: + for energy_category_id, actual_value in aggregated_value['meta_data'].items(): + add_values += " (" + str(tenant['id']) + "," + add_values += " " + str(energy_category_id) + "," + add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "'," + add_values += str(actual_value) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_energy_db.execute(add_values[:-2]) + cnx_energy_db.commit() + + except Exception as e: + error_string = "Error in step 10.1 of tenant_energy_input_category.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + else: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() diff --git a/myems-aggregation/tenant_energy_input_item.py b/myems-aggregation/tenant_energy_input_item.py new file mode 100644 index 00000000..76b76268 --- /dev/null +++ b/myems-aggregation/tenant_energy_input_item.py @@ -0,0 +1,524 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal +import mysql.connector +from multiprocessing import Pool +import random +import config + + +######################################################################################################################## +# PROCEDURES +# Step 1: get all tenants +# Step 2: Create multiprocessing pool to call worker in parallel +######################################################################################################################## + + +def main(logger): + + while True: + # the outermost while loop + ################################################################################################################ + # Step 1: get all tenants + ################################################################################################################ + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + logger.error("Error in step 1.1 of tenant_energy_input_item.main " + str(e)) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + print("Connected to MyEMS System Database") + + try: + cursor_system_db.execute(" SELECT id, name " + " FROM tbl_tenants " + " ORDER BY id ") + rows_tenants = cursor_system_db.fetchall() + + if rows_tenants is None or len(rows_tenants) == 0: + print("There isn't any tenants ") + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + + tenant_list = list() + for row in rows_tenants: + tenant_list.append({"id": row[0], "name": row[1]}) + + except Exception as e: + logger.error("Error in step 1.2 of tenant_energy_input_item.main " + str(e)) + # sleep and continue the outer loop to reconnect the database + time.sleep(60) + continue + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + print("Got all tenants in MyEMS System Database") + + # shuffle the tenant list for randomly calculating the meter hourly value + random.shuffle(tenant_list) + + ################################################################################################################ + # Step 2: Create multiprocessing pool to call worker in parallel + ################################################################################################################ + p = Pool(processes=config.pool_size) + error_list = p.map(worker, tenant_list) + p.close() + p.join() + + for error in error_list: + if error is not None and len(error) > 0: + logger.error(error) + + print("go to sleep 300 seconds...") + time.sleep(300) + print("wake from sleep, and continue to work...") + # end of outer while + + +######################################################################################################################## +# PROCEDURES: +# Step 1: get all input meters associated with the tenant +# Step 2: get all input virtual meters associated with the tenant +# Step 3: get all input offline meters associated with the tenant +# Step 4: determine start datetime and end datetime to aggregate +# Step 5: for each meter in list, get energy input data from energy database +# Step 6: for each virtual meter in list, get energy input data from energy database +# Step 7: for each offline meter in list, get energy input data from energy database +# Step 8: determine common time slot to aggregate +# Step 9: aggregate energy data in the common time slot by energy items and hourly +# Step 10: save energy data to energy database +# +# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter +######################################################################################################################## + +def worker(tenant): + #################################################################################################################### + # Step 1: get all input meters associated with the tenant + #################################################################################################################### + print("Step 1: get all input meters associated with the tenant " + str(tenant['name'])) + + meter_list = list() + cnx_system_db = None + cursor_system_db = None + try: + cnx_system_db = mysql.connector.connect(**config.myems_system_db) + cursor_system_db = cnx_system_db.cursor() + except Exception as e: + error_string = "Error in step 1.1 of tenant_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_meters m, tbl_tenants_meters tm " + " WHERE m.id = tm.meter_id " + " AND m.is_counted = true " + " AND m.energy_item_id is NOT NULL " + " AND tm.tenant_id = %s ", + (tenant['id'],)) + rows_meters = cursor_system_db.fetchall() + + if rows_meters is not None and len(rows_meters) > 0: + for row in rows_meters: + meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 1.2 of tenant_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 2: get all input virtual meters associated with the tenant + #################################################################################################################### + print("Step 2: get all input virtual meters associated with the tenant") + virtual_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_virtual_meters m, tbl_tenants_virtual_meters tm " + " WHERE m.id = tm.virtual_meter_id " + " AND m.energy_item_id is NOT NULL " + " AND m.is_counted = true " + " AND tm.tenant_id = %s ", + (tenant['id'],)) + rows_virtual_meters = cursor_system_db.fetchall() + + if rows_virtual_meters is not None and len(rows_virtual_meters) > 0: + for row in rows_virtual_meters: + virtual_meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 2.1 of tenant_energy_input_item.worker " + str(e) + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 3: get all input offline meters associated with the tenant + #################################################################################################################### + print("Step 3: get all input offline meters associated with the tenant") + + offline_meter_list = list() + + try: + cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id " + " FROM tbl_offline_meters m, tbl_tenants_offline_meters tm " + " WHERE m.id = tm.offline_meter_id " + " AND m.energy_item_id is NOT NULL " + " AND m.is_counted = true " + " AND tm.tenant_id = %s ", + (tenant['id'],)) + rows_offline_meters = cursor_system_db.fetchall() + + if rows_offline_meters is not None and len(rows_offline_meters) > 0: + for row in rows_offline_meters: + offline_meter_list.append({"id": row[0], + "name": row[1], + "energy_item_id": row[2]}) + + except Exception as e: + error_string = "Error in step 3.1 of tenant_energy_input_item.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_system_db: + cursor_system_db.close() + if cnx_system_db: + cnx_system_db.close() + + #################################################################################################################### + # stop to the next tenant if this tenant is empty + #################################################################################################################### + if (meter_list is None or len(meter_list) == 0) and \ + (virtual_meter_list is None or len(virtual_meter_list) == 0) and \ + (offline_meter_list is None or len(offline_meter_list) == 0): + print("This is an empty tenant ") + return None + + #################################################################################################################### + # Step 4: determine start datetime and end datetime to aggregate + #################################################################################################################### + print("Step 4: determine start datetime and end datetime to aggregate") + cnx_energy_db = None + cursor_energy_db = None + try: + cnx_energy_db = mysql.connector.connect(**config.myems_energy_db) + cursor_energy_db = cnx_energy_db.cursor() + except Exception as e: + error_string = "Error in step 4.1 of tenant_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + try: + query = (" SELECT MAX(start_datetime_utc) " + " FROM tbl_tenant_input_item_hourly " + " WHERE tenant_id = %s ") + cursor_energy_db.execute(query, (tenant['id'],)) + row_datetime = cursor_energy_db.fetchone() + start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S') + start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None) + + if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime): + # replace second and microsecond with 0 + # note: do not replace minute in case of calculating in half hourly + start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None) + # start from the next time slot + start_datetime_utc += timedelta(minutes=config.minutes_to_count) + + end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None) + + print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19] + + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19]) + + except Exception as e: + error_string = "Error in step 4.2 of tenant_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 5: for each meter in list, get energy input data from energy database + #################################################################################################################### + energy_meter_hourly = dict() + try: + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_meter_hourly " + " WHERE meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_meter_hourly[meter_id] = None + else: + energy_meter_hourly[meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 5.1 of tenant_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 6: for each virtual meter in list, get energy input data from energy database + #################################################################################################################### + energy_virtual_meter_hourly = dict() + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + try: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_virtual_meter_hourly " + " WHERE virtual_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_virtual_meter_hourly[virtual_meter_id] = None + else: + energy_virtual_meter_hourly[virtual_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1] + except Exception as e: + error_string = "Error in step 6.1 of tenant_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 7: for each offline meter in list, get energy input data from energy database + #################################################################################################################### + energy_offline_meter_hourly = dict() + if offline_meter_list is not None and len(offline_meter_list) > 0: + try: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + + query = (" SELECT start_datetime_utc, actual_value " + " FROM tbl_offline_meter_hourly " + " WHERE offline_meter_id = %s " + " AND start_datetime_utc >= %s " + " AND start_datetime_utc < %s " + " ORDER BY start_datetime_utc ") + cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,)) + rows_energy_values = cursor_energy_db.fetchall() + if rows_energy_values is None or len(rows_energy_values) == 0: + energy_offline_meter_hourly[offline_meter_id] = None + else: + energy_offline_meter_hourly[offline_meter_id] = dict() + for row_energy_value in rows_energy_values: + energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1] + + except Exception as e: + error_string = "Error in step 7.1 of tenant_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 8: determine common time slot to aggregate + #################################################################################################################### + + common_start_datetime_utc = start_datetime_utc + common_end_datetime_utc = end_datetime_utc + + print("Getting common time slot of energy values for all meters") + if energy_meter_hourly is not None and len(energy_meter_hourly) > 0: + for meter_id, energy_hourly in energy_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all virtual meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0: + for meter_id, energy_hourly in energy_virtual_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + print("Getting common time slot of energy values for all offline meters") + if common_start_datetime_utc is not None and common_start_datetime_utc is not None: + if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0: + for meter_id, energy_hourly in energy_offline_meter_hourly.items(): + if energy_hourly is None or len(energy_hourly) == 0: + common_start_datetime_utc = None + common_end_datetime_utc = None + break + else: + if common_start_datetime_utc < min(energy_hourly.keys()): + common_start_datetime_utc = min(energy_hourly.keys()) + if common_end_datetime_utc > max(energy_hourly.keys()): + common_end_datetime_utc = max(energy_hourly.keys()) + + if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \ + (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \ + (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0): + # There isn't any energy data + print("There isn't any energy data") + # continue the for tenant loop to the next tenant + print("continue the for tenant loop to the next tenant") + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + return None + + print("common_start_datetime_utc: " + str(common_start_datetime_utc)) + print("common_end_datetime_utc: " + str(common_end_datetime_utc)) + + #################################################################################################################### + # Step 9: aggregate energy data in the common time slot by energy items and hourly + #################################################################################################################### + + print("Step 9: aggregate energy data in the common time slot by energy items and hourly") + aggregated_values = list() + try: + current_datetime_utc = common_start_datetime_utc + while common_start_datetime_utc is not None \ + and common_end_datetime_utc is not None \ + and current_datetime_utc <= common_end_datetime_utc: + aggregated_value = dict() + aggregated_value['start_datetime_utc'] = current_datetime_utc + aggregated_value['meta_data'] = dict() + + if meter_list is not None and len(meter_list) > 0: + for meter in meter_list: + meter_id = str(meter['id']) + energy_item_id = meter['energy_item_id'] + actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if virtual_meter_list is not None and len(virtual_meter_list) > 0: + for virtual_meter in virtual_meter_list: + virtual_meter_id = str(virtual_meter['id']) + energy_item_id = virtual_meter['energy_item_id'] + actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + if offline_meter_list is not None and len(offline_meter_list) > 0: + for offline_meter in offline_meter_list: + offline_meter_id = str(offline_meter['id']) + energy_item_id = offline_meter['energy_item_id'] + actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0)) + aggregated_value['meta_data'][energy_item_id] = \ + aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value + + aggregated_values.append(aggregated_value) + + current_datetime_utc += timedelta(minutes=config.minutes_to_count) + + except Exception as e: + error_string = "Error in step 9 of tenant_energy_input_item.worker " + str(e) + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + print(error_string) + return error_string + + #################################################################################################################### + # Step 10: save energy data to energy database + #################################################################################################################### + print("Step 10: save energy data to energy database") + + if len(aggregated_values) > 0: + try: + add_values = (" INSERT INTO tbl_tenant_input_item_hourly " + " (tenant_id, " + " energy_item_id, " + " start_datetime_utc, " + " actual_value) " + " VALUES ") + + for aggregated_value in aggregated_values: + for energy_item_id, actual_value in aggregated_value['meta_data'].items(): + add_values += " (" + str(tenant['id']) + "," + add_values += " " + str(energy_item_id) + "," + add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "'," + add_values += str(actual_value) + "), " + print("add_values:" + add_values) + # trim ", " at the end of string and then execute + cursor_energy_db.execute(add_values[:-2]) + cnx_energy_db.commit() + + except Exception as e: + error_string = "Error in step 10.1 of tenant_energy_input_item.worker " + str(e) + print(error_string) + return error_string + finally: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() + else: + if cursor_energy_db: + cursor_energy_db.close() + if cnx_energy_db: + cnx_energy_db.close() diff --git a/myems-aggregation/test_tariff.py b/myems-aggregation/test_tariff.py new file mode 100644 index 00000000..ae3030ec --- /dev/null +++ b/myems-aggregation/test_tariff.py @@ -0,0 +1,33 @@ +from datetime import datetime +import tariff + + +def main(): + """main""" + print('Testing get_energy_category_tariffs ...') + cost_center_id = 1 + energy_category_id = 1 + start_date_time_utc = datetime.strptime('2020-06-29 16:00:00', '%Y-%m-%d %H:%M:%S') + end_date_time_utc = datetime.strptime('2020-07-01 15:59:59', '%Y-%m-%d %H:%M:%S') + tariffs = tariff.get_energy_category_tariffs(cost_center_id, + energy_category_id, + start_date_time_utc, + end_date_time_utc) + for k, v in sorted(tariffs.items()): + print(k, v) + + print('Testing get_energy_item_tariffs ...') + cost_center_id = 1 + energy_item_id = 1 + start_date_time_utc = datetime.strptime('2020-06-29 16:00:00', '%Y-%m-%d %H:%M:%S') + end_date_time_utc = datetime.strptime('2020-07-01 15:59:59', '%Y-%m-%d %H:%M:%S') + tariffs = tariff.get_energy_item_tariffs(cost_center_id, + energy_item_id, + start_date_time_utc, + end_date_time_utc) + for k, v in sorted(tariffs.items()): + print(k, v) + + +if __name__ == "__main__": + main()