merged myems-aggregation

myems-aggregation/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 MyEMS
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

myems-aggregation/README.md

@@ -0,0 +1,62 @@
+## MyEMS Aggregation Service 数据汇总服务
+
+
+
+### Introduction
+
+This service is a component of MyEMS and it aggregates normalized data up to multiple dimensions.
+
+[![Codacy Badge](https://api.codacy.com/project/badge/Grade/cb75cee835ba46118115e088f8be6d87)](https://app.codacy.com/gh/myems/myems-aggregation?utm_source=github.com&utm_medium=referral&utm_content=myems/myems-aggregation&utm_campaign=Badge_Grade)
+[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/myems/myems-aggregation/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/myems/myems-aggregation/?branch=master)
+[![Maintainability](https://api.codeclimate.com/v1/badges/ecff11174fd74975946c/maintainability)](https://codeclimate.com/github/myems/myems-aggregation/maintainability)
+[![Total alerts](https://img.shields.io/lgtm/alerts/g/myems/myems-bacnet.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/myems/myems-bacnet/alerts/)
+
+### Prerequisites
+
+mysql.connector
+
+
+
+### Installation
+
+Download and install MySQL Connector:
+```
+    $ cd ~/tools
+    $ wget https://dev.mysql.com/get/Downloads/Connector-Python/mysql-connector-python-8.0.20.tar.gz
+    $ tar xzf mysql-connector-python-8.0.20.tar.gz
+    $ cd ~/tools/mysql-connector-python-8.0.20
+    $ sudo python3 setup.py install
+```
+
+Install myems-aggregation service:
+```
+    $ cd ~
+    $ git clone https://github.com/myems/myems.git
+    $ cd myems
+    $ sudo git checkout master (or the latest release tag)
+    $ sudo cp -R ~/myems/myems-aggregation /myems-aggregation
+```
+    Edit config.py for your project
+```
+    $ sudo nano /myems-aggregation/config.py
+```
+
+    Setup systemd service:
+```
+    $ sudo cp myems-aggregation.service /lib/systemd/system/
+```
+    Enable the service:
+```
+    $ sudo systemctl enable myems-aggregation.service
+```
+
+    Start the service:
+```
+    $ sudo systemctl start myems-aggregation.service
+```
+
+### References
+
+[1]. https://myems.io
+
+[2]. https://dev.mysql.com/doc/connector-python/en/

myems-aggregation/combined_equipment_billing_input_category.py

@@ -0,0 +1,270 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+import tariff
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all combined equipments
+# for each combined equipment in list:
+#   Step 2: get the latest start_datetime_utc
+#   Step 3: get all energy input data since the latest start_datetime_utc
+#   Step 4: get tariffs
+#   Step 5: calculate billing by multiplying energy with tariff
+#   Step 6: save billing data to database
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all combined equipments
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of combined_equipment_billing_input_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name, cost_center_id "
+                                     " FROM tbl_combined_equipments "
+                                     " ORDER BY id ")
+            rows_combined_equipments = cursor_system_db.fetchall()
+
+            if rows_combined_equipments is None or len(rows_combined_equipments) == 0:
+                print("Step 1.2: There isn't any combined equipments. ")
+                if cursor_system_db:
+                    cursor_system_db.close()
+                if cnx_system_db:
+                    cnx_system_db.close()
+                # sleep and continue the outermost while loop
+                time.sleep(60)
+                continue
+
+            combined_equipment_list = list()
+            for row in rows_combined_equipments:
+                combined_equipment_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of combined_equipment_billing_input_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Step 1.2: Got all combined_equipments from MyEMS System Database")
+
+        cnx_energy_db = None
+        cursor_energy_db = None
+        try:
+            cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+            cursor_energy_db = cnx_energy_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.3 of combined_equipment_billing_input_category " + str(e))
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Energy Database")
+
+        cnx_billing_db = None
+        cursor_billing_db = None
+        try:
+            cnx_billing_db = mysql.connector.connect(**config.myems_billing_db)
+            cursor_billing_db = cnx_billing_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.4 of combined_equipment_billing_input_category " + str(e))
+            if cursor_billing_db:
+                cursor_billing_db.close()
+            if cnx_billing_db:
+                cnx_billing_db.close()
+
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Billing Database")
+
+        for combined_equipment in combined_equipment_list:
+
+            ############################################################################################################
+            # Step 2: get the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 2: get the latest start_datetime_utc from billing database for " + combined_equipment['name'])
+            try:
+                cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) "
+                                          " FROM tbl_combined_equipment_input_category_hourly "
+                                          " WHERE combined_equipment_id = %s ",
+                                          (combined_equipment['id'], ))
+                row_datetime = cursor_billing_db.fetchone()
+                start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+                start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+                if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+                    # replace second and microsecond with 0
+                    # note: do not replace minute in case of calculating in half hourly
+                    start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+                    # start from the next time slot
+                    start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+                print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19])
+            except Exception as e:
+                logger.error("Error in step 2 of combined_equipment_billing_input_category " + str(e))
+                # break the for combined equipment loop
+                break
+
+            ############################################################################################################
+            # Step 3: get all energy input data since the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 3: get all energy input data since the latest start_datetime_utc")
+
+            query = (" SELECT start_datetime_utc, energy_category_id, actual_value "
+                     " FROM tbl_combined_equipment_input_category_hourly "
+                     " WHERE combined_equipment_id = %s AND start_datetime_utc >= %s "
+                     " ORDER BY id ")
+            cursor_energy_db.execute(query, (combined_equipment['id'], start_datetime_utc, ))
+            rows_hourly = cursor_energy_db.fetchall()
+
+            if rows_hourly is None or len(rows_hourly) == 0:
+                print("Step 3: There isn't any energy input data to calculate. ")
+                # continue the for combined equipment loop
+                continue
+
+            energy_dict = dict()
+            energy_category_list = list()
+            end_datetime_utc = start_datetime_utc
+            for row_hourly in rows_hourly:
+                current_datetime_utc = row_hourly[0]
+                energy_category_id = row_hourly[1]
+
+                if energy_category_id not in energy_category_list:
+                    energy_category_list.append(energy_category_id)
+
+                actual_value = row_hourly[2]
+                if energy_dict.get(current_datetime_utc) is None:
+                    energy_dict[current_datetime_utc] = dict()
+                energy_dict[current_datetime_utc][energy_category_id] = actual_value
+                if current_datetime_utc > end_datetime_utc:
+                    end_datetime_utc = current_datetime_utc
+
+            ############################################################################################################
+            # Step 4: get tariffs
+            ############################################################################################################
+            print("Step 4: get tariffs")
+            tariff_dict = dict()
+            for energy_category_id in energy_category_list:
+                tariff_dict[energy_category_id] = \
+                    tariff.get_energy_category_tariffs(combined_equipment['cost_center_id'],
+                                                       energy_category_id,
+                                                       start_datetime_utc,
+                                                       end_datetime_utc)
+            ############################################################################################################
+            # Step 5: calculate billing by multiplying energy with tariff
+            ############################################################################################################
+            print("Step 5: calculate billing by multiplying energy with tariff")
+            billing_dict = dict()
+
+            if len(energy_dict) > 0:
+                for current_datetime_utc in energy_dict.keys():
+                    billing_dict[current_datetime_utc] = dict()
+                    for energy_category_id in energy_category_list:
+                        current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc)
+                        current_energy = energy_dict[current_datetime_utc].get(energy_category_id)
+                        if current_tariff is not None \
+                                and isinstance(current_tariff, Decimal) \
+                                and current_energy is not None \
+                                and isinstance(current_energy, Decimal):
+                            billing_dict[current_datetime_utc][energy_category_id] = \
+                                current_energy * current_tariff
+
+                    if len(billing_dict[current_datetime_utc]) == 0:
+                        del billing_dict[current_datetime_utc]
+
+            ############################################################################################################
+            # Step 6: save billing data to billing database
+            ############################################################################################################
+            print("Step 6: save billing data to billing database")
+
+            if len(billing_dict) > 0:
+                try:
+                    add_values = (" INSERT INTO tbl_combined_equipment_input_category_hourly "
+                                  "             (combined_equipment_id, "
+                                  "              energy_category_id, "
+                                  "              start_datetime_utc, "
+                                  "              actual_value) "
+                                  " VALUES  ")
+
+                    for current_datetime_utc in billing_dict:
+                        for energy_category_id in energy_category_list:
+                            current_billing = billing_dict[current_datetime_utc].get(energy_category_id)
+                            if current_billing is not None and isinstance(current_billing, Decimal):
+                                add_values += " (" + str(combined_equipment['id']) + ","
+                                add_values += " " + str(energy_category_id) + ","
+                                add_values += "'" + current_datetime_utc.isoformat()[0:19] + "',"
+                                add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), "
+                    print("add_values:" + add_values)
+                    # trim ", " at the end of string and then execute
+                    cursor_billing_db.execute(add_values[:-2])
+                    cnx_billing_db.commit()
+                except Exception as e:
+                    logger.error("Error in step 6 of combined_equipment_billing_input_category " + str(e))
+                    # break the for combined equipment loop
+                    break
+
+        # end of for combined equipment loop
+        if cnx_system_db:
+            cnx_system_db.close()
+        if cursor_system_db:
+            cursor_system_db.close()
+
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        if cursor_energy_db:
+            cursor_energy_db.close()
+
+        if cnx_billing_db:
+            cnx_billing_db.close()
+        if cursor_billing_db:
+            cursor_billing_db.close()
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of the outermost while loop

myems-aggregation/combined_equipment_billing_input_item.py

@@ -0,0 +1,270 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+import tariff
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all combined equipments
+# for each combined equipment in list:
+#   Step 2: get the latest start_datetime_utc
+#   Step 3: get all energy input data since the latest start_datetime_utc
+#   Step 4: get tariffs
+#   Step 5: calculate billing by multiplying energy with tariff
+#   Step 6: save billing data to database
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all combined equipments
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of combined_equipment_billing_input_item " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name, cost_center_id "
+                                     " FROM tbl_combined_equipments "
+                                     " ORDER BY id ")
+            rows_combined_equipments = cursor_system_db.fetchall()
+
+            if rows_combined_equipments is None or len(rows_combined_equipments) == 0:
+                print("Step 1.2: There isn't any combined equipments. ")
+                if cursor_system_db:
+                    cursor_system_db.close()
+                if cnx_system_db:
+                    cnx_system_db.close()
+                # sleep and continue the outermost while loop
+                time.sleep(60)
+                continue
+
+            combined_equipment_list = list()
+            for row in rows_combined_equipments:
+                combined_equipment_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of combined_equipment_billing_input_item " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Step 1.2: Got all combined equipments from MyEMS System Database")
+
+        cnx_energy_db = None
+        cursor_energy_db = None
+        try:
+            cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+            cursor_energy_db = cnx_energy_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.3 of combined_equipment_billing_input_item " + str(e))
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Energy Database")
+
+        cnx_billing_db = None
+        cursor_billing_db = None
+        try:
+            cnx_billing_db = mysql.connector.connect(**config.myems_billing_db)
+            cursor_billing_db = cnx_billing_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.4 of combined_equipment_billing_input_item " + str(e))
+            if cursor_billing_db:
+                cursor_billing_db.close()
+            if cnx_billing_db:
+                cnx_billing_db.close()
+
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Billing Database")
+
+        for combined_equipment in combined_equipment_list:
+
+            ############################################################################################################
+            # Step 2: get the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 2: get the latest start_datetime_utc from billing database for " + combined_equipment['name'])
+            try:
+                cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) "
+                                          " FROM tbl_combined_equipment_input_item_hourly "
+                                          " WHERE combined_equipment_id = %s ",
+                                          (combined_equipment['id'], ))
+                row_datetime = cursor_billing_db.fetchone()
+                start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+                start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+                if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+                    # replace second and microsecond with 0
+                    # note: do not replace minute in case of calculating in half hourly
+                    start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+                    # start from the next time slot
+                    start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+                print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19])
+            except Exception as e:
+                logger.error("Error in step 2 of combined_equipment_billing_input_item " + str(e))
+                # break the for combined equipment loop
+                break
+
+            ############################################################################################################
+            # Step 3: get all energy input data since the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 3: get all energy input data since the latest start_datetime_utc")
+
+            query = (" SELECT start_datetime_utc, energy_item_id, actual_value "
+                     " FROM tbl_combined_equipment_input_item_hourly "
+                     " WHERE combined_equipment_id = %s AND start_datetime_utc >= %s "
+                     " ORDER BY id ")
+            cursor_energy_db.execute(query, (combined_equipment['id'], start_datetime_utc, ))
+            rows_hourly = cursor_energy_db.fetchall()
+
+            if rows_hourly is None or len(rows_hourly) == 0:
+                print("Step 3: There isn't any energy input data to calculate. ")
+                # continue the for combined equipment loop
+                continue
+
+            energy_dict = dict()
+            energy_item_list = list()
+            end_datetime_utc = start_datetime_utc
+            for row_hourly in rows_hourly:
+                current_datetime_utc = row_hourly[0]
+                energy_item_id = row_hourly[1]
+
+                if energy_item_id not in energy_item_list:
+                    energy_item_list.append(energy_item_id)
+
+                actual_value = row_hourly[2]
+                if energy_dict.get(current_datetime_utc) is None:
+                    energy_dict[current_datetime_utc] = dict()
+                energy_dict[current_datetime_utc][energy_item_id] = actual_value
+                if current_datetime_utc > end_datetime_utc:
+                    end_datetime_utc = current_datetime_utc
+
+            ############################################################################################################
+            # Step 4: get tariffs
+            ############################################################################################################
+            print("Step 4: get tariffs")
+            tariff_dict = dict()
+            for energy_item_id in energy_item_list:
+                tariff_dict[energy_item_id] = \
+                    tariff.get_energy_item_tariffs(combined_equipment['cost_center_id'],
+                                                   energy_item_id,
+                                                   start_datetime_utc,
+                                                   end_datetime_utc)
+            ############################################################################################################
+            # Step 5: calculate billing by multiplying energy with tariff
+            ############################################################################################################
+            print("Step 5: calculate billing by multiplying energy with tariff")
+            billing_dict = dict()
+
+            if len(energy_dict) > 0:
+                for current_datetime_utc in energy_dict.keys():
+                    billing_dict[current_datetime_utc] = dict()
+                    for energy_item_id in energy_item_list:
+                        current_tariff = tariff_dict[energy_item_id].get(current_datetime_utc)
+                        current_energy = energy_dict[current_datetime_utc].get(energy_item_id)
+                        if current_tariff is not None \
+                                and isinstance(current_tariff, Decimal) \
+                                and current_energy is not None \
+                                and isinstance(current_energy, Decimal):
+                            billing_dict[current_datetime_utc][energy_item_id] = \
+                                current_energy * current_tariff
+
+                    if len(billing_dict[current_datetime_utc]) == 0:
+                        del billing_dict[current_datetime_utc]
+
+            ############################################################################################################
+            # Step 6: save billing data to billing database
+            ############################################################################################################
+            print("Step 6: save billing data to billing database")
+
+            if len(billing_dict) > 0:
+                try:
+                    add_values = (" INSERT INTO tbl_combined_equipment_input_item_hourly "
+                                  "             (combined_equipment_id, "
+                                  "              energy_item_id, "
+                                  "              start_datetime_utc, "
+                                  "              actual_value) "
+                                  " VALUES  ")
+
+                    for current_datetime_utc in billing_dict:
+                        for energy_item_id in energy_item_list:
+                            current_billing = billing_dict[current_datetime_utc].get(energy_item_id)
+                            if current_billing is not None and isinstance(current_billing, Decimal):
+                                add_values += " (" + str(combined_equipment['id']) + ","
+                                add_values += " " + str(energy_item_id) + ","
+                                add_values += "'" + current_datetime_utc.isoformat()[0:19] + "',"
+                                add_values += str(billing_dict[current_datetime_utc][energy_item_id]) + "), "
+                    print("add_values:" + add_values)
+                    # trim ", " at the end of string and then execute
+                    cursor_billing_db.execute(add_values[:-2])
+                    cnx_billing_db.commit()
+                except Exception as e:
+                    logger.error("Error in step 6 of combined_equipment_billing_input_item " + str(e))
+                    # break the for combined equipment loop
+                    break
+
+        # end of for combined equipment loop
+        if cnx_system_db:
+            cnx_system_db.close()
+        if cursor_system_db:
+            cursor_system_db.close()
+
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        if cursor_energy_db:
+            cursor_energy_db.close()
+
+        if cnx_billing_db:
+            cnx_billing_db.close()
+        if cursor_billing_db:
+            cursor_billing_db.close()
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of the outermost while loop

myems-aggregation/combined_equipment_billing_output_category.py

@@ -0,0 +1,270 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+import tariff
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all combined equipments
+# for each combined equipment in list:
+#   Step 2: get the latest start_datetime_utc
+#   Step 3: get all energy input data since the latest start_datetime_utc
+#   Step 4: get tariffs
+#   Step 5: calculate billing by multiplying energy with tariff
+#   Step 6: save billing data to database
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all combined equipments
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of combined_equipment_billing_input_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name, cost_center_id "
+                                     " FROM tbl_combined_equipments "
+                                     " ORDER BY id ")
+            rows_combined_equipments = cursor_system_db.fetchall()
+
+            if rows_combined_equipments is None or len(rows_combined_equipments) == 0:
+                print("Step 1.2: There isn't any combined equipments. ")
+                if cursor_system_db:
+                    cursor_system_db.close()
+                if cnx_system_db:
+                    cnx_system_db.close()
+                # sleep and continue the outermost while loop
+                time.sleep(60)
+                continue
+
+            combined_equipment_list = list()
+            for row in rows_combined_equipments:
+                combined_equipment_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of combined_equipment_billing_output_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Step 1.2: Got all combined equipments from MyEMS System Database")
+
+        cnx_energy_db = None
+        cursor_energy_db = None
+        try:
+            cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+            cursor_energy_db = cnx_energy_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.3 of combined_equipment_billing_output_category " + str(e))
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Energy Database")
+
+        cnx_billing_db = None
+        cursor_billing_db = None
+        try:
+            cnx_billing_db = mysql.connector.connect(**config.myems_billing_db)
+            cursor_billing_db = cnx_billing_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.4 of combined_equipment_billing_output_category " + str(e))
+            if cursor_billing_db:
+                cursor_billing_db.close()
+            if cnx_billing_db:
+                cnx_billing_db.close()
+
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Billing Database")
+
+        for combined_equipment in combined_equipment_list:
+
+            ############################################################################################################
+            # Step 2: get the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 2: get the latest start_datetime_utc from billing database for " + combined_equipment['name'])
+            try:
+                cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) "
+                                          " FROM tbl_combined_equipment_output_category_hourly "
+                                          " WHERE combined_equipment_id = %s ",
+                                          (combined_equipment['id'], ))
+                row_datetime = cursor_billing_db.fetchone()
+                start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+                start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+                if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+                    # replace second and microsecond with 0
+                    # note: do not replace minute in case of calculating in half hourly
+                    start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+                    # start from the next time slot
+                    start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+                print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19])
+            except Exception as e:
+                logger.error("Error in step 2 of combined_equipment_billing_output_category " + str(e))
+                # break the for combined equipment loop
+                break
+
+            ############################################################################################################
+            # Step 3: get all energy output data since the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 3: get all energy output data since the latest start_datetime_utc")
+
+            query = (" SELECT start_datetime_utc, energy_category_id, actual_value "
+                     " FROM tbl_combined_equipment_output_category_hourly "
+                     " WHERE combined_equipment_id = %s AND start_datetime_utc >= %s "
+                     " ORDER BY id ")
+            cursor_energy_db.execute(query, (combined_equipment['id'], start_datetime_utc, ))
+            rows_hourly = cursor_energy_db.fetchall()
+
+            if rows_hourly is None or len(rows_hourly) == 0:
+                print("Step 3: There isn't any energy output data to calculate. ")
+                # continue the for combined equipment loop
+                continue
+
+            energy_dict = dict()
+            energy_category_list = list()
+            end_datetime_utc = start_datetime_utc
+            for row_hourly in rows_hourly:
+                current_datetime_utc = row_hourly[0]
+                energy_category_id = row_hourly[1]
+
+                if energy_category_id not in energy_category_list:
+                    energy_category_list.append(energy_category_id)
+
+                actual_value = row_hourly[2]
+                if energy_dict.get(current_datetime_utc) is None:
+                    energy_dict[current_datetime_utc] = dict()
+                energy_dict[current_datetime_utc][energy_category_id] = actual_value
+                if current_datetime_utc > end_datetime_utc:
+                    end_datetime_utc = current_datetime_utc
+
+            ############################################################################################################
+            # Step 4: get tariffs
+            ############################################################################################################
+            print("Step 4: get tariffs")
+            tariff_dict = dict()
+            for energy_category_id in energy_category_list:
+                tariff_dict[energy_category_id] = \
+                    tariff.get_energy_category_tariffs(combined_equipment['cost_center_id'],
+                                                       energy_category_id,
+                                                       start_datetime_utc,
+                                                       end_datetime_utc)
+            ############################################################################################################
+            # Step 5: calculate billing by multiplying energy with tariff
+            ############################################################################################################
+            print("Step 5: calculate billing by multiplying energy with tariff")
+            billing_dict = dict()
+
+            if len(energy_dict) > 0:
+                for current_datetime_utc in energy_dict.keys():
+                    billing_dict[current_datetime_utc] = dict()
+                    for energy_category_id in energy_category_list:
+                        current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc)
+                        current_energy = energy_dict[current_datetime_utc].get(energy_category_id)
+                        if current_tariff is not None \
+                                and isinstance(current_tariff, Decimal) \
+                                and current_energy is not None \
+                                and isinstance(current_energy, Decimal):
+                            billing_dict[current_datetime_utc][energy_category_id] = \
+                                current_energy * current_tariff
+
+                    if len(billing_dict[current_datetime_utc]) == 0:
+                        del billing_dict[current_datetime_utc]
+
+            ############################################################################################################
+            # Step 6: save billing data to billing database
+            ############################################################################################################
+            print("Step 6: save billing data to billing database")
+
+            if len(billing_dict) > 0:
+                try:
+                    add_values = (" INSERT INTO tbl_combined_equipment_output_category_hourly "
+                                  "             (combined_equipment_id, "
+                                  "              energy_category_id, "
+                                  "              start_datetime_utc, "
+                                  "              actual_value) "
+                                  " VALUES  ")
+
+                    for current_datetime_utc in billing_dict:
+                        for energy_category_id in energy_category_list:
+                            current_billing = billing_dict[current_datetime_utc].get(energy_category_id)
+                            if current_billing is not None and isinstance(current_billing, Decimal):
+                                add_values += " (" + str(combined_equipment['id']) + ","
+                                add_values += " " + str(energy_category_id) + ","
+                                add_values += "'" + current_datetime_utc.isoformat()[0:19] + "',"
+                                add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), "
+                    print("add_values:" + add_values)
+                    # trim ", " at the end of string and then execute
+                    cursor_billing_db.execute(add_values[:-2])
+                    cnx_billing_db.commit()
+                except Exception as e:
+                    logger.error("Error in step 6 of combined_equipment_billing_output_category " + str(e))
+                    # break the for combined_equipment loop
+                    break
+
+        # end of for combined equipment loop
+        if cnx_system_db:
+            cnx_system_db.close()
+        if cursor_system_db:
+            cursor_system_db.close()
+
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        if cursor_energy_db:
+            cursor_energy_db.close()
+
+        if cnx_billing_db:
+            cnx_billing_db.close()
+        if cursor_billing_db:
+            cursor_billing_db.close()
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of the outermost while loop

myems-aggregation/combined_equipment_energy_input_category.py

@@ -0,0 +1,619 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+from multiprocessing import Pool
+import random
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all combined equipments
+# Step 2: Create multiprocessing pool to call worker in parallel
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all combined equipments
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of combined_equipment_energy_input_category.main " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name "
+                                     " FROM tbl_combined_equipments "
+                                     " ORDER BY id ")
+            rows_combined_equipments = cursor_system_db.fetchall()
+
+            if rows_combined_equipments is None or len(rows_combined_equipments) == 0:
+                print("There isn't any combined equipments ")
+                # sleep and continue the outer loop to reconnect the database
+                time.sleep(60)
+                continue
+
+            combined_equipment_list = list()
+            for row in rows_combined_equipments:
+                combined_equipment_list.append({"id": row[0], "name": row[1]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of combined_equipment_energy_input_category.main " + str(e))
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        finally:
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+
+        print("Got all combined equipments in MyEMS System Database")
+
+        # shuffle the combined equipment list for randomly calculating the meter hourly value
+        random.shuffle(combined_equipment_list)
+
+        ################################################################################################################
+        # Step 2: Create multiprocessing pool to call worker in parallel
+        ################################################################################################################
+        p = Pool(processes=config.pool_size)
+        error_list = p.map(worker, combined_equipment_list)
+        p.close()
+        p.join()
+
+        for error in error_list:
+            if error is not None and len(error) > 0:
+                logger.error(error)
+
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of outer while
+
+
+########################################################################################################################
+# PROCEDURES:
+#   Step 1: get all input meters associated with the combined equipment
+#   Step 2: get all input virtual meters associated with the combined equipment
+#   Step 3: get all input offline meters associated with the combined equipment
+#   Step 4: get all equipments associated with the combined equipment
+#   Step 5: determine start datetime and end datetime to aggregate
+#   Step 6: for each meter in list, get energy input data from energy database
+#   Step 7: for each virtual meter in list, get energy input data from energy database
+#   Step 8: for each offline meter in list, get energy input data from energy database
+#   Step 9: for each equipment in list, get energy input data from energy database
+#   Step 10: determine common time slot to aggregate
+#   Step 11: aggregate energy data in the common time slot by energy categories and hourly
+#   Step 12: save energy data to energy database
+#
+# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter
+########################################################################################################################
+
+def worker(combined_equipment):
+    ####################################################################################################################
+    # Step 1: get all input meters associated with the combined equipment
+    ####################################################################################################################
+    print("Step 1: get all input meters associated with the combined equipment " + str(combined_equipment['name']))
+
+    meter_list = list()
+    cnx_system_db = None
+    cursor_system_db = None
+    try:
+        cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+        cursor_system_db = cnx_system_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 1.1 of combined_equipment_energy_input_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_meters m, tbl_combined_equipments_meters em "
+                                 " WHERE m.id = em.meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND em.is_output = false "
+                                 "       AND em.combined_equipment_id = %s ",
+                                 (combined_equipment['id'],))
+        rows_meters = cursor_system_db.fetchall()
+
+        if rows_meters is not None and len(rows_meters) > 0:
+            for row in rows_meters:
+                meter_list.append({"id": row[0],
+                                   "name": row[1],
+                                   "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 1.2 of combined_equipment_energy_input_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 2: get all input virtual meters associated with the combined equipment
+    ####################################################################################################################
+    print("Step 2: get all input virtual meters associated with the combined equipment")
+    virtual_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_virtual_meters m, tbl_combined_equipments_virtual_meters em "
+                                 " WHERE m.id = em.virtual_meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND em.is_output = false "
+                                 "       AND em.combined_equipment_id = %s ",
+                                 (combined_equipment['id'],))
+        rows_virtual_meters = cursor_system_db.fetchall()
+
+        if rows_virtual_meters is not None and len(rows_virtual_meters) > 0:
+            for row in rows_virtual_meters:
+                virtual_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 2.1 of combined_equipment_energy_input_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 3: get all input offline meters associated with the combined equipment
+    ####################################################################################################################
+    print("Step 3: get all input offline meters associated with the combined equipment")
+
+    offline_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_offline_meters m, tbl_combined_equipments_offline_meters em "
+                                 " WHERE m.id = em.offline_meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND em.is_output = false "
+                                 "       AND em.combined_equipment_id = %s ",
+                                 (combined_equipment['id'],))
+        rows_offline_meters = cursor_system_db.fetchall()
+
+        if rows_offline_meters is not None and len(rows_offline_meters) > 0:
+            for row in rows_offline_meters:
+                offline_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 3.1 of combined_equipment_energy_input_category.worker " + str(e)
+        print(error_string)
+        return error_string
+    finally:
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+
+    ####################################################################################################################
+    # Step 4: get all equipments associated with the combined equipment
+    ####################################################################################################################
+    print("Step 4: get all equipments associated with the combined equipment")
+
+    equipment_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT e.id, e.name "
+                                 " FROM tbl_equipments e, tbl_combined_equipments_equipments ce "
+                                 " WHERE e.id = ce.equipment_id "
+                                 "       AND e.is_input_counted = true "
+                                 "       AND ce.combined_equipment_id = %s ",
+                                 (combined_equipment['id'],))
+        rows_equipments = cursor_system_db.fetchall()
+
+        if rows_equipments is not None and len(rows_equipments) > 0:
+            for row in rows_equipments:
+                equipment_list.append({"id": row[0],
+                                       "name": row[1]})
+
+    except Exception as e:
+        error_string = "Error in step 4 of combined_equipment_energy_input_category.worker " + str(e)
+        print(error_string)
+        return error_string
+    finally:
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+
+    ####################################################################################################################
+    # stop to the next combined equipment if this combined equipment is empty
+    ####################################################################################################################
+    if (meter_list is None or len(meter_list) == 0) and \
+            (virtual_meter_list is None or len(virtual_meter_list) == 0) and \
+            (offline_meter_list is None or len(offline_meter_list) == 0) and \
+            (equipment_list is None or len(equipment_list) == 0):
+        print("This is an empty combined equipment ")
+        return None
+
+    ####################################################################################################################
+    # Step 5: determine start datetime and end datetime to aggregate
+    ####################################################################################################################
+    print("Step 5: determine start datetime and end datetime to aggregate")
+    cnx_energy_db = None
+    cursor_energy_db = None
+    try:
+        cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+        cursor_energy_db = cnx_energy_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 5.1 of combined_equipment_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        query = (" SELECT MAX(start_datetime_utc) "
+                 " FROM tbl_combined_equipment_input_category_hourly "
+                 " WHERE combined_equipment_id = %s ")
+        cursor_energy_db.execute(query, (combined_equipment['id'],))
+        row_datetime = cursor_energy_db.fetchone()
+        start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+        start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+        if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+            # replace second and microsecond with 0
+            # note: do not replace minute in case of calculating in half hourly
+            start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+            # start from the next time slot
+            start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+        end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None)
+
+        print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]
+              + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19])
+
+    except Exception as e:
+        error_string = "Error in step 5.2 of combined_equipment_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 6: for each meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_meter_hourly = dict()
+    try:
+        if meter_list is not None and len(meter_list) > 0:
+            for meter in meter_list:
+                meter_id = str(meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_meter_hourly "
+                         " WHERE meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_meter_hourly[meter_id] = None
+                else:
+                    energy_meter_hourly[meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1]
+    except Exception as e:
+        error_string = "Error in step 6.1 of combined_equipment_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 7: for each virtual meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_virtual_meter_hourly = dict()
+    if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+        try:
+            for virtual_meter in virtual_meter_list:
+                virtual_meter_id = str(virtual_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_virtual_meter_hourly "
+                         " WHERE virtual_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_virtual_meter_hourly[virtual_meter_id] = None
+                else:
+                    energy_virtual_meter_hourly[virtual_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1]
+        except Exception as e:
+            error_string = "Error in step 7.1 of combined_equipment_energy_input_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 8: for each offline meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_offline_meter_hourly = dict()
+    if offline_meter_list is not None and len(offline_meter_list) > 0:
+        try:
+            for offline_meter in offline_meter_list:
+                offline_meter_id = str(offline_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_offline_meter_hourly "
+                         " WHERE offline_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_offline_meter_hourly[offline_meter_id] = None
+                else:
+                    energy_offline_meter_hourly[offline_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1]
+
+        except Exception as e:
+            error_string = "Error in step 8.1 of combined_equipment_energy_input_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 9: for each equipment in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_equipment_hourly = dict()
+    if equipment_list is not None and len(equipment_list) > 0:
+        try:
+            for equipment in equipment_list:
+                equipment_id = str(equipment['id'])
+                query = (" SELECT start_datetime_utc, energy_category_id, actual_value "
+                         " FROM tbl_equipment_input_category_hourly "
+                         " WHERE equipment_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (equipment_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_equipment_hourly[equipment_id] = None
+                else:
+                    energy_equipment_hourly[equipment_id] = dict()
+                    for row_value in rows_energy_values:
+                        current_datetime_utc = row_value[0]
+                        if current_datetime_utc not in energy_equipment_hourly[equipment_id]:
+                            energy_equipment_hourly[equipment_id][current_datetime_utc] = dict()
+                        energy_category_id = row_value[1]
+                        actual_value = row_value[2]
+                        energy_equipment_hourly[equipment_id][current_datetime_utc][energy_category_id] = \
+                            actual_value
+        except Exception as e:
+            error_string = "Error in step 9 of combined_equipment_energy_input_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 10: determine common time slot to aggregate
+    ####################################################################################################################
+
+    common_start_datetime_utc = start_datetime_utc
+    common_end_datetime_utc = end_datetime_utc
+
+    print("Getting common time slot of energy values for all meters")
+    if energy_meter_hourly is not None and len(energy_meter_hourly) > 0:
+        for meter_id, energy_hourly in energy_meter_hourly.items():
+            if energy_hourly is None or len(energy_hourly) == 0:
+                common_start_datetime_utc = None
+                common_end_datetime_utc = None
+                break
+            else:
+                if common_start_datetime_utc < min(energy_hourly.keys()):
+                    common_start_datetime_utc = min(energy_hourly.keys())
+                if common_end_datetime_utc > max(energy_hourly.keys()):
+                    common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all virtual meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_virtual_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all offline meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_offline_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all equipments...")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_equipment_hourly is not None and len(energy_equipment_hourly) > 0:
+            for equipment_id, energy_hourly in energy_equipment_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \
+            (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \
+            (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0) and \
+            (energy_equipment_hourly is None or len(energy_equipment_hourly) == 0):
+        # There isn't any energy data
+        print("There isn't any energy data")
+        # continue the for combined equipment loop to the next combined equipment
+        print("continue the for combined equipment loop to the next combined equipment")
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        return None
+
+    print("common_start_datetime_utc: " + str(common_start_datetime_utc))
+    print("common_end_datetime_utc: " + str(common_end_datetime_utc))
+
+    ####################################################################################################################
+    # Step 11: aggregate energy data in the common time slot by energy categories and hourly
+    ####################################################################################################################
+
+    print("Step 11: aggregate energy data in the common time slot by energy categories and hourly")
+    aggregated_values = list()
+    try:
+        current_datetime_utc = common_start_datetime_utc
+        while common_start_datetime_utc is not None \
+                and common_end_datetime_utc is not None \
+                and current_datetime_utc <= common_end_datetime_utc:
+            aggregated_value = dict()
+            aggregated_value['start_datetime_utc'] = current_datetime_utc
+            aggregated_value['meta_data'] = dict()
+
+            if meter_list is not None and len(meter_list) > 0:
+                for meter in meter_list:
+                    meter_id = str(meter['id'])
+                    energy_category_id = meter['energy_category_id']
+                    actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+                for virtual_meter in virtual_meter_list:
+                    virtual_meter_id = str(virtual_meter['id'])
+                    energy_category_id = virtual_meter['energy_category_id']
+                    actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if offline_meter_list is not None and len(offline_meter_list) > 0:
+                for offline_meter in offline_meter_list:
+                    offline_meter_id = str(offline_meter['id'])
+                    energy_category_id = offline_meter['energy_category_id']
+                    actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if equipment_list is not None and len(equipment_list) > 0:
+                for equipment in equipment_list:
+                    equipment_id = str(equipment['id'])
+                    meta_data_dict = energy_equipment_hourly[equipment_id].get(current_datetime_utc, None)
+                    if meta_data_dict is not None and len(meta_data_dict) > 0:
+                        for energy_category_id, actual_value in meta_data_dict.items():
+                            aggregated_value['meta_data'][energy_category_id] = \
+                                aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            aggregated_values.append(aggregated_value)
+
+            current_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+    except Exception as e:
+        error_string = "Error in step 11 of combined_equipment_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 12: save energy data to energy database
+    ####################################################################################################################
+    print("Step 12: save energy data to energy database")
+
+    if len(aggregated_values) > 0:
+        try:
+            add_values = (" INSERT INTO tbl_combined_equipment_input_category_hourly "
+                          "             (combined_equipment_id, "
+                          "              energy_category_id, "
+                          "              start_datetime_utc, "
+                          "              actual_value) "
+                          " VALUES  ")
+
+            for aggregated_value in aggregated_values:
+                for energy_category_id, actual_value in aggregated_value['meta_data'].items():
+                    add_values += " (" + str(combined_equipment['id']) + ","
+                    add_values += " " + str(energy_category_id) + ","
+                    add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "',"
+                    add_values += str(actual_value) + "), "
+            print("add_values:" + add_values)
+            # trim ", " at the end of string and then execute
+            cursor_energy_db.execute(add_values[:-2])
+            cnx_energy_db.commit()
+
+        except Exception as e:
+            error_string = "Error in step 12.1 of combined_equipment_energy_input_category.worker " + str(e)
+            print(error_string)
+            return error_string
+        finally:
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+    else:
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()

myems-aggregation/combined_equipment_energy_input_item.py

@@ -0,0 +1,622 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+from multiprocessing import Pool
+import random
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all combined equipments
+# Step 2: Create multiprocessing pool to call worker in parallel
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all combined equipments
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of combined_equipment_energy_input_item.main " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name "
+                                     " FROM tbl_combined_equipments "
+                                     " ORDER BY id ")
+            rows_combined_equipments = cursor_system_db.fetchall()
+
+            if rows_combined_equipments is None or len(rows_combined_equipments) == 0:
+                print("There isn't any combined equipments ")
+                # sleep and continue the outer loop to reconnect the database
+                time.sleep(60)
+                continue
+
+            combined_equipment_list = list()
+            for row in rows_combined_equipments:
+                combined_equipment_list.append({"id": row[0], "name": row[1]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of combined_equipment_energy_input_item.main " + str(e))
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        finally:
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+
+        print("Got all combined equipments in MyEMS System Database")
+
+        # shuffle the combined equipment list for randomly calculating the meter hourly value
+        random.shuffle(combined_equipment_list)
+
+        ################################################################################################################
+        # Step 2: Create multiprocessing pool to call worker in parallel
+        ################################################################################################################
+        p = Pool(processes=config.pool_size)
+        error_list = p.map(worker, combined_equipment_list)
+        p.close()
+        p.join()
+
+        for error in error_list:
+            if error is not None and len(error) > 0:
+                logger.error(error)
+
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of outer while
+
+
+########################################################################################################################
+# PROCEDURES:
+#   Step 1: get all input meters associated with the combined equipment
+#   Step 2: get all input virtual meters associated with the combined equipment
+#   Step 3: get all input offline meters associated with the combined equipment
+#   Step 4: get all equipments associated with the combined equipment
+#   Step 5: determine start datetime and end datetime to aggregate
+#   Step 6: for each meter in list, get energy input data from energy database
+#   Step 7: for each virtual meter in list, get energy input data from energy database
+#   Step 8: for each offline meter in list, get energy input data from energy database
+#   Step 9: for each equipment in list, get energy input data from energy database
+#   Step 10: determine common time slot to aggregate
+#   Step 11: aggregate energy data in the common time slot by energy items and hourly
+#   Step 12: save energy data to energy database
+#
+# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter
+########################################################################################################################
+
+def worker(combined_equipment):
+    ####################################################################################################################
+    # Step 1: get all input meters associated with the combined equipment
+    ####################################################################################################################
+    print("Step 1: get all input meters associated with the combined equipment " + str(combined_equipment['name']))
+
+    meter_list = list()
+    cnx_system_db = None
+    cursor_system_db = None
+    try:
+        cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+        cursor_system_db = cnx_system_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 1.1 of combined_equipment_energy_input_item.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id "
+                                 " FROM tbl_meters m, tbl_combined_equipments_meters em "
+                                 " WHERE m.id = em.meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND m.energy_item_id is NOT NULL "
+                                 "       AND em.is_output = false "
+                                 "       AND em.combined_equipment_id = %s ",
+                                 (combined_equipment['id'],))
+        rows_meters = cursor_system_db.fetchall()
+
+        if rows_meters is not None and len(rows_meters) > 0:
+            for row in rows_meters:
+                meter_list.append({"id": row[0],
+                                   "name": row[1],
+                                   "energy_item_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 1.2 of combined_equipment_energy_input_item.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 2: get all input virtual meters associated with the combined equipment
+    ####################################################################################################################
+    print("Step 2: get all input virtual meters associated with the combined equipment")
+    virtual_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id "
+                                 " FROM tbl_virtual_meters m, tbl_combined_equipments_virtual_meters em "
+                                 " WHERE m.id = em.virtual_meter_id "
+                                 "       AND m.energy_item_id is NOT NULL "
+                                 "       AND m.is_counted = true "
+                                 "       AND em.is_output = false "
+                                 "       AND em.combined_equipment_id = %s ",
+                                 (combined_equipment['id'],))
+        rows_virtual_meters = cursor_system_db.fetchall()
+
+        if rows_virtual_meters is not None and len(rows_virtual_meters) > 0:
+            for row in rows_virtual_meters:
+                virtual_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_item_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 2.1 of combined_equipment_energy_input_item.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 3: get all input offline meters associated with the combined equipment
+    ####################################################################################################################
+    print("Step 3: get all input offline meters associated with the combined equipment")
+
+    offline_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id "
+                                 " FROM tbl_offline_meters m, tbl_combined_equipments_offline_meters em "
+                                 " WHERE m.id = em.offline_meter_id "
+                                 "       AND m.energy_item_id is NOT NULL "
+                                 "       AND m.is_counted = true "
+                                 "       AND em.is_output = false "
+                                 "       AND em.combined_equipment_id = %s ",
+                                 (combined_equipment['id'],))
+        rows_offline_meters = cursor_system_db.fetchall()
+
+        if rows_offline_meters is not None and len(rows_offline_meters) > 0:
+            for row in rows_offline_meters:
+                offline_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_item_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 3.1 of combined_equipment_energy_input_item.worker " + str(e)
+        print(error_string)
+        return error_string
+    finally:
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+
+    ####################################################################################################################
+    # Step 4: get all equipments associated with the combined equipment
+    ####################################################################################################################
+    print("Step 4: get all equipments associated with the combined equipment")
+
+    equipment_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT e.id, e.name "
+                                 " FROM tbl_equipments e, tbl_combined_equipments_equipments ce "
+                                 " WHERE e.id = ce.equipment_id "
+                                 "       AND e.is_input_counted = true "
+                                 "       AND ce.combined_equipment_id = %s ",
+                                 (combined_equipment['id'],))
+        rows_equipments = cursor_system_db.fetchall()
+
+        if rows_equipments is not None and len(rows_equipments) > 0:
+            for row in rows_equipments:
+                equipment_list.append({"id": row[0],
+                                       "name": row[1]})
+
+    except Exception as e:
+        error_string = "Error in step 4 of combined_equipment_energy_input_item.worker " + str(e)
+        print(error_string)
+        return error_string
+    finally:
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+
+    ####################################################################################################################
+    # stop to the next combined equipment if this combined equipment is empty
+    ####################################################################################################################
+    if (meter_list is None or len(meter_list) == 0) and \
+            (virtual_meter_list is None or len(virtual_meter_list) == 0) and \
+            (offline_meter_list is None or len(offline_meter_list) == 0) and \
+            (equipment_list is None or len(equipment_list) == 0):
+        print("This is an empty combined equipment ")
+        return None
+
+    ####################################################################################################################
+    # Step 5: determine start datetime and end datetime to aggregate
+    ####################################################################################################################
+    print("Step 5: determine start datetime and end datetime to aggregate")
+    cnx_energy_db = None
+    cursor_energy_db = None
+    try:
+        cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+        cursor_energy_db = cnx_energy_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 5.1 of combined_equipment_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        query = (" SELECT MAX(start_datetime_utc) "
+                 " FROM tbl_combined_equipment_input_item_hourly "
+                 " WHERE combined_equipment_id = %s ")
+        cursor_energy_db.execute(query, (combined_equipment['id'],))
+        row_datetime = cursor_energy_db.fetchone()
+        start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+        start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+        if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+            # replace second and microsecond with 0
+            # note: do not replace minute in case of calculating in half hourly
+            start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+            # start from the next time slot
+            start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+        end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None)
+
+        print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]
+              + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19])
+
+    except Exception as e:
+        error_string = "Error in step 5.2 of combined_equipment_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 6: for each meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_meter_hourly = dict()
+    try:
+        if meter_list is not None and len(meter_list) > 0:
+            for meter in meter_list:
+                meter_id = str(meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_meter_hourly "
+                         " WHERE meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_meter_hourly[meter_id] = None
+                else:
+                    energy_meter_hourly[meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1]
+    except Exception as e:
+        error_string = "Error in step 6.1 of combined_equipment_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 7: for each virtual meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_virtual_meter_hourly = dict()
+    if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+        try:
+            for virtual_meter in virtual_meter_list:
+                virtual_meter_id = str(virtual_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_virtual_meter_hourly "
+                         " WHERE virtual_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_virtual_meter_hourly[virtual_meter_id] = None
+                else:
+                    energy_virtual_meter_hourly[virtual_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1]
+        except Exception as e:
+            error_string = "Error in step 7.1 of combined_equipment_energy_input_item.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 8: for each offline meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_offline_meter_hourly = dict()
+    if offline_meter_list is not None and len(offline_meter_list) > 0:
+        try:
+            for offline_meter in offline_meter_list:
+                offline_meter_id = str(offline_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_offline_meter_hourly "
+                         " WHERE offline_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_offline_meter_hourly[offline_meter_id] = None
+                else:
+                    energy_offline_meter_hourly[offline_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1]
+
+        except Exception as e:
+            error_string = "Error in step 8.1 of combined_equipment_energy_input_item.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 9: for each equipment in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_equipment_hourly = dict()
+    if equipment_list is not None and len(equipment_list) > 0:
+        try:
+            for equipment in equipment_list:
+                equipment_id = str(equipment['id'])
+                query = (" SELECT start_datetime_utc, energy_category_id, actual_value "
+                         " FROM tbl_equipment_input_item_hourly "
+                         " WHERE equipment_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (equipment_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_equipment_hourly[equipment_id] = None
+                else:
+                    energy_equipment_hourly[equipment_id] = dict()
+                    for row_value in rows_energy_values:
+                        current_datetime_utc = row_value[0]
+                        if current_datetime_utc not in energy_equipment_hourly[equipment_id]:
+                            energy_equipment_hourly[equipment_id][current_datetime_utc] = dict()
+                        energy_category_id = row_value[1]
+                        actual_value = row_value[2]
+                        energy_equipment_hourly[equipment_id][current_datetime_utc][energy_category_id] = \
+                            actual_value
+        except Exception as e:
+            error_string = "Error in step 9 of combined_equipment_energy_input_item.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 10: determine common time slot to aggregate
+    ####################################################################################################################
+
+    common_start_datetime_utc = start_datetime_utc
+    common_end_datetime_utc = end_datetime_utc
+
+    print("Getting common time slot of energy values for all meters")
+    if energy_meter_hourly is not None and len(energy_meter_hourly) > 0:
+        for meter_id, energy_hourly in energy_meter_hourly.items():
+            if energy_hourly is None or len(energy_hourly) == 0:
+                common_start_datetime_utc = None
+                common_end_datetime_utc = None
+                break
+            else:
+                if common_start_datetime_utc < min(energy_hourly.keys()):
+                    common_start_datetime_utc = min(energy_hourly.keys())
+                if common_end_datetime_utc > max(energy_hourly.keys()):
+                    common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all virtual meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_virtual_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all offline meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_offline_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all equipments")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_equipment_hourly is not None and len(energy_equipment_hourly) > 0:
+            for equipment_id, energy_hourly in energy_equipment_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \
+            (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \
+            (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0) and \
+            (energy_equipment_hourly is None or len(energy_equipment_hourly) == 0):
+        # There isn't any energy data
+        print("There isn't any energy data")
+        # continue the for combined equipment loop to the next combined equipment
+        print("continue the for combined equipment loop to the next combined equipment")
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        return None
+
+    print("common_start_datetime_utc: " + str(common_start_datetime_utc))
+    print("common_end_datetime_utc: " + str(common_end_datetime_utc))
+
+    ####################################################################################################################
+    # Step 11: aggregate energy data in the common time slot by energy items and hourly
+    ####################################################################################################################
+
+    print("Step 11: aggregate energy data in the common time slot by energy items and hourly")
+    aggregated_values = list()
+    try:
+        current_datetime_utc = common_start_datetime_utc
+        while common_start_datetime_utc is not None \
+                and common_end_datetime_utc is not None \
+                and current_datetime_utc <= common_end_datetime_utc:
+            aggregated_value = dict()
+            aggregated_value['start_datetime_utc'] = current_datetime_utc
+            aggregated_value['meta_data'] = dict()
+
+            if meter_list is not None and len(meter_list) > 0:
+                for meter in meter_list:
+                    meter_id = str(meter['id'])
+                    energy_item_id = meter['energy_item_id']
+                    actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_item_id] = \
+                        aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value
+
+            if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+                for virtual_meter in virtual_meter_list:
+                    virtual_meter_id = str(virtual_meter['id'])
+                    energy_item_id = virtual_meter['energy_item_id']
+                    actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_item_id] = \
+                        aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value
+
+            if offline_meter_list is not None and len(offline_meter_list) > 0:
+                for offline_meter in offline_meter_list:
+                    offline_meter_id = str(offline_meter['id'])
+                    energy_item_id = offline_meter['energy_item_id']
+                    actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_item_id] = \
+                        aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value
+
+            if equipment_list is not None and len(equipment_list) > 0:
+                for equipment in equipment_list:
+                    equipment_id = str(equipment['id'])
+                    meta_data_dict = energy_equipment_hourly[equipment_id].get(current_datetime_utc, None)
+                    if meta_data_dict is not None and len(meta_data_dict) > 0:
+                        for energy_category_id, actual_value in meta_data_dict.items():
+                            aggregated_value['meta_data'][energy_category_id] = \
+                                aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            aggregated_values.append(aggregated_value)
+
+            current_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+    except Exception as e:
+        error_string = "Error in step 11 of combined_equipment_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 12: save energy data to energy database
+    ####################################################################################################################
+    print("Step 12: save energy data to energy database")
+
+    if len(aggregated_values) > 0:
+        try:
+            add_values = (" INSERT INTO tbl_combined_equipment_input_item_hourly "
+                          "             (combined_equipment_id, "
+                          "              energy_item_id, "
+                          "              start_datetime_utc, "
+                          "              actual_value) "
+                          " VALUES  ")
+
+            for aggregated_value in aggregated_values:
+                for energy_item_id, actual_value in aggregated_value['meta_data'].items():
+                    add_values += " (" + str(combined_equipment['id']) + ","
+                    add_values += " " + str(energy_item_id) + ","
+                    add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "',"
+                    add_values += str(actual_value) + "), "
+            print("add_values:" + add_values)
+            # trim ", " at the end of string and then execute
+            cursor_energy_db.execute(add_values[:-2])
+            cnx_energy_db.commit()
+
+        except Exception as e:
+            error_string = "Error in step 12.1 of combined_equipment_energy_input_item.worker " + str(e)
+            print(error_string)
+            return error_string
+        finally:
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+    else:
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()

myems-aggregation/combined_equipment_energy_output_category.py

@@ -0,0 +1,619 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+from multiprocessing import Pool
+import random
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all combined equipments
+# Step 2: Create multiprocessing pool to call worker in parallel
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all combined equipments
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of combined_equipment_energy_output_category.main " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name "
+                                     " FROM tbl_combined_equipments "
+                                     " ORDER BY id ")
+            rows_combined_equipments = cursor_system_db.fetchall()
+
+            if rows_combined_equipments is None or len(rows_combined_equipments) == 0:
+                print("There isn't any combined equipments ")
+                # sleep and continue the outer loop to reconnect the database
+                time.sleep(60)
+                continue
+
+            combined_equipment_list = list()
+            for row in rows_combined_equipments:
+                combined_equipment_list.append({"id": row[0], "name": row[1]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of combined_equipment_energy_output_category.main " + str(e))
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        finally:
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+
+        print("Got all combined equipments in MyEMS System Database")
+
+        # shuffle the combined equipment list for randomly calculating the meter hourly value
+        random.shuffle(combined_equipment_list)
+
+        ################################################################################################################
+        # Step 2: Create multiprocessing pool to call worker in parallel
+        ################################################################################################################
+        p = Pool(processes=config.pool_size)
+        error_list = p.map(worker, combined_equipment_list)
+        p.close()
+        p.join()
+
+        for error in error_list:
+            if error is not None and len(error) > 0:
+                logger.error(error)
+
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of outer while
+
+
+########################################################################################################################
+# PROCEDURES:
+#   Step 1: get all output meters associated with the combined equipment
+#   Step 2: get all output virtual meters associated with the combined equipment
+#   Step 3: get all output offline meters associated with the combined equipment
+#   Step 4: get all equipments associated with the combined equipment
+#   Step 5: determine start datetime and end datetime to aggregate
+#   Step 6: for each meter in list, get energy output data from energy database
+#   Step 7: for each virtual meter in list, get energy output data from energy database
+#   Step 8: for each offline meter in list, get energy output data from energy database
+#   Step 9: for each equipment in list, get energy output data from energy database
+#   Step 10: determine common time slot to aggregate
+#   Step 11: aggregate energy data in the common time slot by energy categories and hourly
+#   Step 12: save energy data to energy database
+#
+# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter
+########################################################################################################################
+
+def worker(combined_equipment):
+    ####################################################################################################################
+    # Step 1: get all output meters associated with the combined equipment
+    ####################################################################################################################
+    print("Step 1: get all output meters associated with the combined equipment " + str(combined_equipment['name']))
+
+    meter_list = list()
+    cnx_system_db = None
+    cursor_system_db = None
+    try:
+        cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+        cursor_system_db = cnx_system_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 1.1 of combined_equipment_energy_output_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_meters m, tbl_combined_equipments_meters em "
+                                 " WHERE m.id = em.meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND em.is_output = true "
+                                 "       AND em.combined_equipment_id = %s ",
+                                 (combined_equipment['id'],))
+        rows_meters = cursor_system_db.fetchall()
+
+        if rows_meters is not None and len(rows_meters) > 0:
+            for row in rows_meters:
+                meter_list.append({"id": row[0],
+                                   "name": row[1],
+                                   "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 1.2 of combined_equipment_energy_output_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 2: get all output virtual meters associated with the combined equipment
+    ####################################################################################################################
+    print("Step 2: get all output virtual meters associated with the combined equipment")
+    virtual_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_virtual_meters m, tbl_combined_equipments_virtual_meters em "
+                                 " WHERE m.id = em.virtual_meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND em.is_output = true "
+                                 "       AND em.combined_equipment_id = %s ",
+                                 (combined_equipment['id'],))
+        rows_virtual_meters = cursor_system_db.fetchall()
+
+        if rows_virtual_meters is not None and len(rows_virtual_meters) > 0:
+            for row in rows_virtual_meters:
+                virtual_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 2.1 of combined_equipment_energy_output_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 3: get all output offline meters associated with the combined equipment
+    ####################################################################################################################
+    print("Step 3: get all output offline meters associated with the combined equipment")
+
+    offline_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_offline_meters m, tbl_combined_equipments_offline_meters em "
+                                 " WHERE m.id = em.offline_meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND em.is_output = true "
+                                 "       AND em.combined_equipment_id = %s ",
+                                 (combined_equipment['id'],))
+        rows_offline_meters = cursor_system_db.fetchall()
+
+        if rows_offline_meters is not None and len(rows_offline_meters) > 0:
+            for row in rows_offline_meters:
+                offline_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 3.1 of combined_equipment_energy_output_category.worker " + str(e)
+        print(error_string)
+        return error_string
+    finally:
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+
+    ####################################################################################################################
+    # Step 4: get all equipments associated with the combined equipment
+    ####################################################################################################################
+    print("Step 4: get all equipments associated with the combined equipment")
+
+    equipment_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT e.id, e.name "
+                                 " FROM tbl_equipments e, tbl_combined_equipments_equipments ce "
+                                 " WHERE e.id = ce.equipment_id "
+                                 "       AND e.is_output_counted = true "
+                                 "       AND ce.combined_equipment_id = %s ",
+                                 (combined_equipment['id'],))
+        rows_equipments = cursor_system_db.fetchall()
+
+        if rows_equipments is not None and len(rows_equipments) > 0:
+            for row in rows_equipments:
+                equipment_list.append({"id": row[0],
+                                       "name": row[1]})
+
+    except Exception as e:
+        error_string = "Error in step 4 of combined_equipment_energy_output_category.worker " + str(e)
+        print(error_string)
+        return error_string
+    finally:
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+
+    ####################################################################################################################
+    # stop to the next combined equipment if this combined 3equipment is empty
+    ####################################################################################################################
+    if (meter_list is None or len(meter_list) == 0) and \
+            (virtual_meter_list is None or len(virtual_meter_list) == 0) and \
+            (offline_meter_list is None or len(offline_meter_list) == 0) and \
+            (equipment_list is None or len(equipment_list) == 0):
+        print("This is an empty combined equipment ")
+        return None
+
+    ####################################################################################################################
+    # Step 5: determine start datetime and end datetime to aggregate
+    ####################################################################################################################
+    print("Step 5: determine start datetime and end datetime to aggregate")
+    cnx_energy_db = None
+    cursor_energy_db = None
+    try:
+        cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+        cursor_energy_db = cnx_energy_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 5.1 of combined_equipment_energy_output_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        query = (" SELECT MAX(start_datetime_utc) "
+                 " FROM tbl_combined_equipment_output_category_hourly "
+                 " WHERE combined_equipment_id = %s ")
+        cursor_energy_db.execute(query, (combined_equipment['id'],))
+        row_datetime = cursor_energy_db.fetchone()
+        start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+        start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+        if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+            # replace second and microsecond with 0
+            # note: do not replace minute in case of calculating in half hourly
+            start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+            # start from the next time slot
+            start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+        end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None)
+
+        print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]
+              + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19])
+
+    except Exception as e:
+        error_string = "Error in step 5.2 of combined_equipment_energy_output_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 6: for each meter in list, get energy output data from energy database
+    ####################################################################################################################
+    energy_meter_hourly = dict()
+    try:
+        if meter_list is not None and len(meter_list) > 0:
+            for meter in meter_list:
+                meter_id = str(meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_meter_hourly "
+                         " WHERE meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_meter_hourly[meter_id] = None
+                else:
+                    energy_meter_hourly[meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1]
+    except Exception as e:
+        error_string = "Error in step 6.1 of combined_equipment_energy_output_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 7: for each virtual meter in list, get energy output data from energy database
+    ####################################################################################################################
+    energy_virtual_meter_hourly = dict()
+    if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+        try:
+            for virtual_meter in virtual_meter_list:
+                virtual_meter_id = str(virtual_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_virtual_meter_hourly "
+                         " WHERE virtual_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_virtual_meter_hourly[virtual_meter_id] = None
+                else:
+                    energy_virtual_meter_hourly[virtual_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1]
+        except Exception as e:
+            error_string = "Error in step 7.1 of combined_equipment_energy_output_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 8: for each offline meter in list, get energy output data from energy database
+    ####################################################################################################################
+    energy_offline_meter_hourly = dict()
+    if offline_meter_list is not None and len(offline_meter_list) > 0:
+        try:
+            for offline_meter in offline_meter_list:
+                offline_meter_id = str(offline_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_offline_meter_hourly "
+                         " WHERE offline_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_offline_meter_hourly[offline_meter_id] = None
+                else:
+                    energy_offline_meter_hourly[offline_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1]
+
+        except Exception as e:
+            error_string = "Error in step 8.1 of combined_equipment_energy_output_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 9: for each equipment in list, get energy output data from energy database
+    ####################################################################################################################
+    energy_equipment_hourly = dict()
+    if equipment_list is not None and len(equipment_list) > 0:
+        try:
+            for equipment in equipment_list:
+                equipment_id = str(equipment['id'])
+                query = (" SELECT start_datetime_utc, energy_category_id, actual_value "
+                         " FROM tbl_equipment_output_category_hourly "
+                         " WHERE equipment_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (equipment_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_equipment_hourly[equipment_id] = None
+                else:
+                    energy_equipment_hourly[equipment_id] = dict()
+                    for row_value in rows_energy_values:
+                        current_datetime_utc = row_value[0]
+                        if current_datetime_utc not in energy_equipment_hourly[equipment_id]:
+                            energy_equipment_hourly[equipment_id][current_datetime_utc] = dict()
+                        energy_category_id = row_value[1]
+                        actual_value = row_value[2]
+                        energy_equipment_hourly[equipment_id][current_datetime_utc][energy_category_id] = \
+                            actual_value
+        except Exception as e:
+            error_string = "Error in step 9 of combined_equipment_energy_output_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 10: determine common time slot to aggregate
+    ####################################################################################################################
+
+    common_start_datetime_utc = start_datetime_utc
+    common_end_datetime_utc = end_datetime_utc
+
+    print("Getting common time slot of energy values for all meters")
+    if energy_meter_hourly is not None and len(energy_meter_hourly) > 0:
+        for meter_id, energy_hourly in energy_meter_hourly.items():
+            if energy_hourly is None or len(energy_hourly) == 0:
+                common_start_datetime_utc = None
+                common_end_datetime_utc = None
+                break
+            else:
+                if common_start_datetime_utc < min(energy_hourly.keys()):
+                    common_start_datetime_utc = min(energy_hourly.keys())
+                if common_end_datetime_utc > max(energy_hourly.keys()):
+                    common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all virtual meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_virtual_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all offline meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_offline_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all equipments...")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_equipment_hourly is not None and len(energy_equipment_hourly) > 0:
+            for equipment_id, energy_hourly in energy_equipment_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \
+            (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \
+            (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0) and \
+            (energy_equipment_hourly is None or len(energy_equipment_hourly) == 0):
+        # There isn't any energy data
+        print("There isn't any energy data")
+        # continue the for combined equipment loop to the next combined equipment
+        print("continue the for combined equipment loop to the next combined equipment")
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        return None
+
+    print("common_start_datetime_utc: " + str(common_start_datetime_utc))
+    print("common_end_datetime_utc: " + str(common_end_datetime_utc))
+
+    ####################################################################################################################
+    # Step 11: aggregate energy data in the common time slot by energy categories and hourly
+    ####################################################################################################################
+
+    print("Step 11: aggregate energy data in the common time slot by energy categories and hourly")
+    aggregated_values = list()
+    try:
+        current_datetime_utc = common_start_datetime_utc
+        while common_start_datetime_utc is not None \
+                and common_end_datetime_utc is not None \
+                and current_datetime_utc <= common_end_datetime_utc:
+            aggregated_value = dict()
+            aggregated_value['start_datetime_utc'] = current_datetime_utc
+            aggregated_value['meta_data'] = dict()
+
+            if meter_list is not None and len(meter_list) > 0:
+                for meter in meter_list:
+                    meter_id = str(meter['id'])
+                    energy_category_id = meter['energy_category_id']
+                    actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+                for virtual_meter in virtual_meter_list:
+                    virtual_meter_id = str(virtual_meter['id'])
+                    energy_category_id = virtual_meter['energy_category_id']
+                    actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if offline_meter_list is not None and len(offline_meter_list) > 0:
+                for offline_meter in offline_meter_list:
+                    offline_meter_id = str(offline_meter['id'])
+                    energy_category_id = offline_meter['energy_category_id']
+                    actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if equipment_list is not None and len(equipment_list) > 0:
+                for equipment in equipment_list:
+                    equipment_id = str(equipment['id'])
+                    meta_data_dict = energy_equipment_hourly[equipment_id].get(current_datetime_utc, None)
+                    if meta_data_dict is not None and len(meta_data_dict) > 0:
+                        for energy_category_id, actual_value in meta_data_dict.items():
+                            aggregated_value['meta_data'][energy_category_id] = \
+                                aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            aggregated_values.append(aggregated_value)
+
+            current_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+    except Exception as e:
+        error_string = "Error in step 11 of combined_equipment_energy_output_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 12: save energy data to energy database
+    ####################################################################################################################
+    print("Step 12: save energy data to energy database")
+
+    if len(aggregated_values) > 0:
+        try:
+            add_values = (" INSERT INTO tbl_combined_equipment_output_category_hourly "
+                          "             (combined_equipment_id, "
+                          "              energy_category_id, "
+                          "              start_datetime_utc, "
+                          "              actual_value) "
+                          " VALUES  ")
+
+            for aggregated_value in aggregated_values:
+                for energy_category_id, actual_value in aggregated_value['meta_data'].items():
+                    add_values += " (" + str(combined_equipment['id']) + ","
+                    add_values += " " + str(energy_category_id) + ","
+                    add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "',"
+                    add_values += str(actual_value) + "), "
+            print("add_values:" + add_values)
+            # trim ", " at the end of string and then execute
+            cursor_energy_db.execute(add_values[:-2])
+            cnx_energy_db.commit()
+
+        except Exception as e:
+            error_string = "Error in step 12.1 of combined_equipment_energy_output_category.worker " + str(e)
+            print(error_string)
+            return error_string
+        finally:
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+    else:
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()

myems-aggregation/config.py

@@ -0,0 +1,45 @@
+myems_system_db = {
+    'user': 'root',
+    'password': '!MyEMS1',
+    'host': '127.0.0.1',
+    'database': 'myems_system_db',
+}
+
+myems_historical_db = {
+    'user': 'root',
+    'password': '!MyEMS1',
+    'host': '127.0.0.1',
+    'database': 'myems_historical_db',
+}
+
+myems_energy_db = {
+    'user': 'root',
+    'password': '!MyEMS1',
+    'host': '127.0.0.1',
+    'database': 'myems_energy_db',
+}
+
+myems_billing_db = {
+    'user': 'root',
+    'password': '!MyEMS1',
+    'host': '127.0.0.1',
+    'database': 'myems_billing_db',
+}
+
+
+# indicates how long in minutes energy data will be aggregated
+# 30 for half hourly
+# 60 for hourly
+minutes_to_count = 60
+
+# indicates from when (in UTC timezone) to recalculate if the energy data is null or were cleared
+# format string: '%Y-%m-%d %H:%M:%S'
+start_datetime_utc = '2019-12-31 16:00:00'
+
+# indicates the project's time zone offset from UTC
+utc_offset = '+08:00'
+
+# the number of worker processes in parallel
+# the pool size depends on the computing performance of the database server and the analysis server
+pool_size = 5
+

myems-aggregation/equipment_billing_input_category.py

@@ -0,0 +1,269 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+import tariff
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all equipments
+# for each equipment in list:
+#   Step 2: get the latest start_datetime_utc
+#   Step 3: get all energy input data since the latest start_datetime_utc
+#   Step 4: get tariffs
+#   Step 5: calculate billing by multiplying energy with tariff
+#   Step 6: save billing data to database
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all equipments
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of equipment_billing_input_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name, cost_center_id "
+                                     " FROM tbl_equipments "
+                                     " ORDER BY id ")
+            rows_equipments = cursor_system_db.fetchall()
+
+            if rows_equipments is None or len(rows_equipments) == 0:
+                print("Step 1.2: There isn't any equipments. ")
+                if cursor_system_db:
+                    cursor_system_db.close()
+                if cnx_system_db:
+                    cnx_system_db.close()
+                # sleep and continue the outermost while loop
+                time.sleep(60)
+                continue
+
+            equipment_list = list()
+            for row in rows_equipments:
+                equipment_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of equipment_billing_input_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Step 1.2: Got all equipments from MyEMS System Database")
+
+        cnx_energy_db = None
+        cursor_energy_db = None
+        try:
+            cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+            cursor_energy_db = cnx_energy_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.3 of equipment_billing_input_category " + str(e))
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Energy Database")
+
+        cnx_billing_db = None
+        cursor_billing_db = None
+        try:
+            cnx_billing_db = mysql.connector.connect(**config.myems_billing_db)
+            cursor_billing_db = cnx_billing_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.4 of equipment_billing_input_category " + str(e))
+            if cursor_billing_db:
+                cursor_billing_db.close()
+            if cnx_billing_db:
+                cnx_billing_db.close()
+
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Billing Database")
+
+        for equipment in equipment_list:
+
+            ############################################################################################################
+            # Step 2: get the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 2: get the latest start_datetime_utc from billing database for " + equipment['name'])
+            try:
+                cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) "
+                                          " FROM tbl_equipment_input_category_hourly "
+                                          " WHERE equipment_id = %s ",
+                                          (equipment['id'], ))
+                row_datetime = cursor_billing_db.fetchone()
+                start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+                start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+                if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+                    # replace second and microsecond with 0
+                    # note: do not replace minute in case of calculating in half hourly
+                    start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+                    # start from the next time slot
+                    start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+                print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19])
+            except Exception as e:
+                logger.error("Error in step 2 of equipment_billing_input_category " + str(e))
+                # break the for equipment loop
+                break
+
+            ############################################################################################################
+            # Step 3: get all energy input data since the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 3: get all energy input data since the latest start_datetime_utc")
+
+            query = (" SELECT start_datetime_utc, energy_category_id, actual_value "
+                     " FROM tbl_equipment_input_category_hourly "
+                     " WHERE equipment_id = %s AND start_datetime_utc >= %s "
+                     " ORDER BY id ")
+            cursor_energy_db.execute(query, (equipment['id'], start_datetime_utc, ))
+            rows_hourly = cursor_energy_db.fetchall()
+
+            if rows_hourly is None or len(rows_hourly) == 0:
+                print("Step 3: There isn't any energy input data to calculate. ")
+                # continue the for equipment loop
+                continue
+
+            energy_dict = dict()
+            energy_category_list = list()
+            end_datetime_utc = start_datetime_utc
+            for row_hourly in rows_hourly:
+                current_datetime_utc = row_hourly[0]
+                energy_category_id = row_hourly[1]
+
+                if energy_category_id not in energy_category_list:
+                    energy_category_list.append(energy_category_id)
+
+                actual_value = row_hourly[2]
+                if energy_dict.get(current_datetime_utc) is None:
+                    energy_dict[current_datetime_utc] = dict()
+                energy_dict[current_datetime_utc][energy_category_id] = actual_value
+                if current_datetime_utc > end_datetime_utc:
+                    end_datetime_utc = current_datetime_utc
+
+            ############################################################################################################
+            # Step 4: get tariffs
+            ############################################################################################################
+            print("Step 4: get tariffs")
+            tariff_dict = dict()
+            for energy_category_id in energy_category_list:
+                tariff_dict[energy_category_id] = tariff.get_energy_category_tariffs(equipment['cost_center_id'],
+                                                                                     energy_category_id,
+                                                                                     start_datetime_utc,
+                                                                                     end_datetime_utc)
+            ############################################################################################################
+            # Step 5: calculate billing by multiplying energy with tariff
+            ############################################################################################################
+            print("Step 5: calculate billing by multiplying energy with tariff")
+            billing_dict = dict()
+
+            if len(energy_dict) > 0:
+                for current_datetime_utc in energy_dict.keys():
+                    billing_dict[current_datetime_utc] = dict()
+                    for energy_category_id in energy_category_list:
+                        current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc)
+                        current_energy = energy_dict[current_datetime_utc].get(energy_category_id)
+                        if current_tariff is not None \
+                                and isinstance(current_tariff, Decimal) \
+                                and current_energy is not None \
+                                and isinstance(current_energy, Decimal):
+                            billing_dict[current_datetime_utc][energy_category_id] = \
+                                current_energy * current_tariff
+
+                    if len(billing_dict[current_datetime_utc]) == 0:
+                        del billing_dict[current_datetime_utc]
+
+            ############################################################################################################
+            # Step 6: save billing data to billing database
+            ############################################################################################################
+            print("Step 6: save billing data to billing database")
+
+            if len(billing_dict) > 0:
+                try:
+                    add_values = (" INSERT INTO tbl_equipment_input_category_hourly "
+                                  "             (equipment_id, "
+                                  "              energy_category_id, "
+                                  "              start_datetime_utc, "
+                                  "              actual_value) "
+                                  " VALUES  ")
+
+                    for current_datetime_utc in billing_dict:
+                        for energy_category_id in energy_category_list:
+                            current_billing = billing_dict[current_datetime_utc].get(energy_category_id)
+                            if current_billing is not None and isinstance(current_billing, Decimal):
+                                add_values += " (" + str(equipment['id']) + ","
+                                add_values += " " + str(energy_category_id) + ","
+                                add_values += "'" + current_datetime_utc.isoformat()[0:19] + "',"
+                                add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), "
+                    print("add_values:" + add_values)
+                    # trim ", " at the end of string and then execute
+                    cursor_billing_db.execute(add_values[:-2])
+                    cnx_billing_db.commit()
+                except Exception as e:
+                    logger.error("Error in step 6 of equipment_billing_input_category " + str(e))
+                    # break the for equipment loop
+                    break
+
+        # end of for equipment loop
+        if cnx_system_db:
+            cnx_system_db.close()
+        if cursor_system_db:
+            cursor_system_db.close()
+
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        if cursor_energy_db:
+            cursor_energy_db.close()
+
+        if cnx_billing_db:
+            cnx_billing_db.close()
+        if cursor_billing_db:
+            cursor_billing_db.close()
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of the outermost while loop

myems-aggregation/equipment_billing_input_item.py

@@ -0,0 +1,269 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+import tariff
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all equipments
+# for each equipment in list:
+#   Step 2: get the latest start_datetime_utc
+#   Step 3: get all energy input data since the latest start_datetime_utc
+#   Step 4: get tariffs
+#   Step 5: calculate billing by multiplying energy with tariff
+#   Step 6: save billing data to database
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all equipments
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of equipment_billing_input_item " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name, cost_center_id "
+                                     " FROM tbl_equipments "
+                                     " ORDER BY id ")
+            rows_equipments = cursor_system_db.fetchall()
+
+            if rows_equipments is None or len(rows_equipments) == 0:
+                print("Step 1.2: There isn't any equipments. ")
+                if cursor_system_db:
+                    cursor_system_db.close()
+                if cnx_system_db:
+                    cnx_system_db.close()
+                # sleep and continue the outermost while loop
+                time.sleep(60)
+                continue
+
+            equipment_list = list()
+            for row in rows_equipments:
+                equipment_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of equipment_billing_input_item " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Step 1.2: Got all equipments from MyEMS System Database")
+
+        cnx_energy_db = None
+        cursor_energy_db = None
+        try:
+            cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+            cursor_energy_db = cnx_energy_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.3 of equipment_billing_input_item " + str(e))
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Energy Database")
+
+        cnx_billing_db = None
+        cursor_billing_db = None
+        try:
+            cnx_billing_db = mysql.connector.connect(**config.myems_billing_db)
+            cursor_billing_db = cnx_billing_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.4 of equipment_billing_input_item " + str(e))
+            if cursor_billing_db:
+                cursor_billing_db.close()
+            if cnx_billing_db:
+                cnx_billing_db.close()
+
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Billing Database")
+
+        for equipment in equipment_list:
+
+            ############################################################################################################
+            # Step 2: get the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 2: get the latest start_datetime_utc from billing database for " + equipment['name'])
+            try:
+                cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) "
+                                          " FROM tbl_equipment_input_item_hourly "
+                                          " WHERE equipment_id = %s ",
+                                          (equipment['id'], ))
+                row_datetime = cursor_billing_db.fetchone()
+                start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+                start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+                if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+                    # replace second and microsecond with 0
+                    # note: do not replace minute in case of calculating in half hourly
+                    start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+                    # start from the next time slot
+                    start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+                print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19])
+            except Exception as e:
+                logger.error("Error in step 2 of equipment_billing_input_item " + str(e))
+                # break the for equipment loop
+                break
+
+            ############################################################################################################
+            # Step 3: get all energy input data since the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 3: get all energy input data since the latest start_datetime_utc")
+
+            query = (" SELECT start_datetime_utc, energy_item_id, actual_value "
+                     " FROM tbl_equipment_input_item_hourly "
+                     " WHERE equipment_id = %s AND start_datetime_utc >= %s "
+                     " ORDER BY id ")
+            cursor_energy_db.execute(query, (equipment['id'], start_datetime_utc, ))
+            rows_hourly = cursor_energy_db.fetchall()
+
+            if rows_hourly is None or len(rows_hourly) == 0:
+                print("Step 3: There isn't any energy input data to calculate. ")
+                # continue the for equipment loop
+                continue
+
+            energy_dict = dict()
+            energy_item_list = list()
+            end_datetime_utc = start_datetime_utc
+            for row_hourly in rows_hourly:
+                current_datetime_utc = row_hourly[0]
+                energy_item_id = row_hourly[1]
+
+                if energy_item_id not in energy_item_list:
+                    energy_item_list.append(energy_item_id)
+
+                actual_value = row_hourly[2]
+                if energy_dict.get(current_datetime_utc) is None:
+                    energy_dict[current_datetime_utc] = dict()
+                energy_dict[current_datetime_utc][energy_item_id] = actual_value
+                if current_datetime_utc > end_datetime_utc:
+                    end_datetime_utc = current_datetime_utc
+
+            ############################################################################################################
+            # Step 4: get tariffs
+            ############################################################################################################
+            print("Step 4: get tariffs")
+            tariff_dict = dict()
+            for energy_item_id in energy_item_list:
+                tariff_dict[energy_item_id] = tariff.get_energy_item_tariffs(equipment['cost_center_id'],
+                                                                             energy_item_id,
+                                                                             start_datetime_utc,
+                                                                             end_datetime_utc)
+            ############################################################################################################
+            # Step 5: calculate billing by multiplying energy with tariff
+            ############################################################################################################
+            print("Step 5: calculate billing by multiplying energy with tariff")
+            billing_dict = dict()
+
+            if len(energy_dict) > 0:
+                for current_datetime_utc in energy_dict.keys():
+                    billing_dict[current_datetime_utc] = dict()
+                    for energy_item_id in energy_item_list:
+                        current_tariff = tariff_dict[energy_item_id].get(current_datetime_utc)
+                        current_energy = energy_dict[current_datetime_utc].get(energy_item_id)
+                        if current_tariff is not None \
+                                and isinstance(current_tariff, Decimal) \
+                                and current_energy is not None \
+                                and isinstance(current_energy, Decimal):
+                            billing_dict[current_datetime_utc][energy_item_id] = \
+                                current_energy * current_tariff
+
+                    if len(billing_dict[current_datetime_utc]) == 0:
+                        del billing_dict[current_datetime_utc]
+
+            ############################################################################################################
+            # Step 6: save billing data to billing database
+            ############################################################################################################
+            print("Step 6: save billing data to billing database")
+
+            if len(billing_dict) > 0:
+                try:
+                    add_values = (" INSERT INTO tbl_equipment_input_item_hourly "
+                                  "             (equipment_id, "
+                                  "              energy_item_id, "
+                                  "              start_datetime_utc, "
+                                  "              actual_value) "
+                                  " VALUES  ")
+
+                    for current_datetime_utc in billing_dict:
+                        for energy_item_id in energy_item_list:
+                            current_billing = billing_dict[current_datetime_utc].get(energy_item_id)
+                            if current_billing is not None and isinstance(current_billing, Decimal):
+                                add_values += " (" + str(equipment['id']) + ","
+                                add_values += " " + str(energy_item_id) + ","
+                                add_values += "'" + current_datetime_utc.isoformat()[0:19] + "',"
+                                add_values += str(billing_dict[current_datetime_utc][energy_item_id]) + "), "
+                    print("add_values:" + add_values)
+                    # trim ", " at the end of string and then execute
+                    cursor_billing_db.execute(add_values[:-2])
+                    cnx_billing_db.commit()
+                except Exception as e:
+                    logger.error("Error in step 6 of equipment_billing_input_item " + str(e))
+                    # break the for equipment loop
+                    break
+
+        # end of for equipment loop
+        if cnx_system_db:
+            cnx_system_db.close()
+        if cursor_system_db:
+            cursor_system_db.close()
+
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        if cursor_energy_db:
+            cursor_energy_db.close()
+
+        if cnx_billing_db:
+            cnx_billing_db.close()
+        if cursor_billing_db:
+            cursor_billing_db.close()
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of the outermost while loop

myems-aggregation/equipment_billing_output_category.py

@@ -0,0 +1,269 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+import tariff
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all equipments
+# for each equipment in list:
+#   Step 2: get the latest start_datetime_utc
+#   Step 3: get all energy input data since the latest start_datetime_utc
+#   Step 4: get tariffs
+#   Step 5: calculate billing by multiplying energy with tariff
+#   Step 6: save billing data to database
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all equipments
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of equipment_billing_input_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name, cost_center_id "
+                                     " FROM tbl_equipments "
+                                     " ORDER BY id ")
+            rows_equipments = cursor_system_db.fetchall()
+
+            if rows_equipments is None or len(rows_equipments) == 0:
+                print("Step 1.2: There isn't any equipments. ")
+                if cursor_system_db:
+                    cursor_system_db.close()
+                if cnx_system_db:
+                    cnx_system_db.close()
+                # sleep and continue the outermost while loop
+                time.sleep(60)
+                continue
+
+            equipment_list = list()
+            for row in rows_equipments:
+                equipment_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of equipment_billing_output_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Step 1.2: Got all equipments from MyEMS System Database")
+
+        cnx_energy_db = None
+        cursor_energy_db = None
+        try:
+            cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+            cursor_energy_db = cnx_energy_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.3 of equipment_billing_output_category " + str(e))
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Energy Database")
+
+        cnx_billing_db = None
+        cursor_billing_db = None
+        try:
+            cnx_billing_db = mysql.connector.connect(**config.myems_billing_db)
+            cursor_billing_db = cnx_billing_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.4 of equipment_billing_output_category " + str(e))
+            if cursor_billing_db:
+                cursor_billing_db.close()
+            if cnx_billing_db:
+                cnx_billing_db.close()
+
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Billing Database")
+
+        for equipment in equipment_list:
+
+            ############################################################################################################
+            # Step 2: get the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 2: get the latest start_datetime_utc from billing database for " + equipment['name'])
+            try:
+                cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) "
+                                          " FROM tbl_equipment_output_category_hourly "
+                                          " WHERE equipment_id = %s ",
+                                          (equipment['id'], ))
+                row_datetime = cursor_billing_db.fetchone()
+                start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+                start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+                if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+                    # replace second and microsecond with 0
+                    # note: do not replace minute in case of calculating in half hourly
+                    start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+                    # start from the next time slot
+                    start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+                print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19])
+            except Exception as e:
+                logger.error("Error in step 2 of equipment_billing_output_category " + str(e))
+                # break the for equipment loop
+                break
+
+            ############################################################################################################
+            # Step 3: get all energy output data since the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 3: get all energy output data since the latest start_datetime_utc")
+
+            query = (" SELECT start_datetime_utc, energy_category_id, actual_value "
+                     " FROM tbl_equipment_output_category_hourly "
+                     " WHERE equipment_id = %s AND start_datetime_utc >= %s "
+                     " ORDER BY id ")
+            cursor_energy_db.execute(query, (equipment['id'], start_datetime_utc, ))
+            rows_hourly = cursor_energy_db.fetchall()
+
+            if rows_hourly is None or len(rows_hourly) == 0:
+                print("Step 3: There isn't any energy output data to calculate. ")
+                # continue the for equipment loop
+                continue
+
+            energy_dict = dict()
+            energy_category_list = list()
+            end_datetime_utc = start_datetime_utc
+            for row_hourly in rows_hourly:
+                current_datetime_utc = row_hourly[0]
+                energy_category_id = row_hourly[1]
+
+                if energy_category_id not in energy_category_list:
+                    energy_category_list.append(energy_category_id)
+
+                actual_value = row_hourly[2]
+                if energy_dict.get(current_datetime_utc) is None:
+                    energy_dict[current_datetime_utc] = dict()
+                energy_dict[current_datetime_utc][energy_category_id] = actual_value
+                if current_datetime_utc > end_datetime_utc:
+                    end_datetime_utc = current_datetime_utc
+
+            ############################################################################################################
+            # Step 4: get tariffs
+            ############################################################################################################
+            print("Step 4: get tariffs")
+            tariff_dict = dict()
+            for energy_category_id in energy_category_list:
+                tariff_dict[energy_category_id] = tariff.get_energy_category_tariffs(equipment['cost_center_id'],
+                                                                                     energy_category_id,
+                                                                                     start_datetime_utc,
+                                                                                     end_datetime_utc)
+            ############################################################################################################
+            # Step 5: calculate billing by multiplying energy with tariff
+            ############################################################################################################
+            print("Step 5: calculate billing by multiplying energy with tariff")
+            billing_dict = dict()
+
+            if len(energy_dict) > 0:
+                for current_datetime_utc in energy_dict.keys():
+                    billing_dict[current_datetime_utc] = dict()
+                    for energy_category_id in energy_category_list:
+                        current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc)
+                        current_energy = energy_dict[current_datetime_utc].get(energy_category_id)
+                        if current_tariff is not None \
+                                and isinstance(current_tariff, Decimal) \
+                                and current_energy is not None \
+                                and isinstance(current_energy, Decimal):
+                            billing_dict[current_datetime_utc][energy_category_id] = \
+                                current_energy * current_tariff
+
+                    if len(billing_dict[current_datetime_utc]) == 0:
+                        del billing_dict[current_datetime_utc]
+
+            ############################################################################################################
+            # Step 6: save billing data to billing database
+            ############################################################################################################
+            print("Step 6: save billing data to billing database")
+
+            if len(billing_dict) > 0:
+                try:
+                    add_values = (" INSERT INTO tbl_equipment_output_category_hourly "
+                                  "             (equipment_id, "
+                                  "              energy_category_id, "
+                                  "              start_datetime_utc, "
+                                  "              actual_value) "
+                                  " VALUES  ")
+
+                    for current_datetime_utc in billing_dict:
+                        for energy_category_id in energy_category_list:
+                            current_billing = billing_dict[current_datetime_utc].get(energy_category_id)
+                            if current_billing is not None and isinstance(current_billing, Decimal):
+                                add_values += " (" + str(equipment['id']) + ","
+                                add_values += " " + str(energy_category_id) + ","
+                                add_values += "'" + current_datetime_utc.isoformat()[0:19] + "',"
+                                add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), "
+                    print("add_values:" + add_values)
+                    # trim ", " at the end of string and then execute
+                    cursor_billing_db.execute(add_values[:-2])
+                    cnx_billing_db.commit()
+                except Exception as e:
+                    logger.error("Error in step 6 of equipment_billing_output_category " + str(e))
+                    # break the for equipment loop
+                    break
+
+        # end of for equipment loop
+        if cnx_system_db:
+            cnx_system_db.close()
+        if cursor_system_db:
+            cursor_system_db.close()
+
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        if cursor_energy_db:
+            cursor_energy_db.close()
+
+        if cnx_billing_db:
+            cnx_billing_db.close()
+        if cursor_billing_db:
+            cursor_billing_db.close()
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of the outermost while loop

myems-aggregation/equipment_energy_input_category.py

@@ -0,0 +1,524 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+from multiprocessing import Pool
+import random
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all equipments
+# Step 2: Create multiprocessing pool to call worker in parallel
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all equipments
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of equipment_energy_input_category.main " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name "
+                                     " FROM tbl_equipments "
+                                     " ORDER BY id ")
+            rows_equipments = cursor_system_db.fetchall()
+
+            if rows_equipments is None or len(rows_equipments) == 0:
+                print("There isn't any equipments ")
+                # sleep and continue the outer loop to reconnect the database
+                time.sleep(60)
+                continue
+
+            equipment_list = list()
+            for row in rows_equipments:
+                equipment_list.append({"id": row[0], "name": row[1]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of equipment_energy_input_category.main " + str(e))
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        finally:
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+
+        print("Got all equipments in MyEMS System Database")
+
+        # shuffle the equipment list for randomly calculating the meter hourly value
+        random.shuffle(equipment_list)
+
+        ################################################################################################################
+        # Step 2: Create multiprocessing pool to call worker in parallel
+        ################################################################################################################
+        p = Pool(processes=config.pool_size)
+        error_list = p.map(worker, equipment_list)
+        p.close()
+        p.join()
+
+        for error in error_list:
+            if error is not None and len(error) > 0:
+                logger.error(error)
+
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of outer while
+
+
+########################################################################################################################
+# PROCEDURES:
+#   Step 1: get all input meters associated with the equipment
+#   Step 2: get all input virtual meters associated with the equipment
+#   Step 3: get all input offline meters associated with the equipment
+#   Step 4: determine start datetime and end datetime to aggregate
+#   Step 5: for each meter in list, get energy input data from energy database
+#   Step 6: for each virtual meter in list, get energy input data from energy database
+#   Step 7: for each offline meter in list, get energy input data from energy database
+#   Step 8: determine common time slot to aggregate
+#   Step 9: aggregate energy data in the common time slot by energy categories and hourly
+#   Step 10: save energy data to energy database
+#
+# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter
+########################################################################################################################
+
+def worker(equipment):
+    ####################################################################################################################
+    # Step 1: get all input meters associated with the equipment
+    ####################################################################################################################
+    print("Step 1: get all input meters associated with the equipment " + str(equipment['name']))
+
+    meter_list = list()
+    cnx_system_db = None
+    cursor_system_db = None
+    try:
+        cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+        cursor_system_db = cnx_system_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 1.1 of equipment_energy_input_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_meters m, tbl_equipments_meters em "
+                                 " WHERE m.id = em.meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND em.is_output = false "
+                                 "       AND em.equipment_id = %s ",
+                                 (equipment['id'],))
+        rows_meters = cursor_system_db.fetchall()
+
+        if rows_meters is not None and len(rows_meters) > 0:
+            for row in rows_meters:
+                meter_list.append({"id": row[0],
+                                   "name": row[1],
+                                   "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 1.2 of equipment_energy_input_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 2: get all input virtual meters associated with the equipment
+    ####################################################################################################################
+    print("Step 2: get all input virtual meters associated with the equipment")
+    virtual_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_virtual_meters m, tbl_equipments_virtual_meters em "
+                                 " WHERE m.id = em.virtual_meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND em.is_output = false "
+                                 "       AND em.equipment_id = %s ",
+                                 (equipment['id'],))
+        rows_virtual_meters = cursor_system_db.fetchall()
+
+        if rows_virtual_meters is not None and len(rows_virtual_meters) > 0:
+            for row in rows_virtual_meters:
+                virtual_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 2.1 of equipment_energy_input_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 3: get all input offline meters associated with the equipment
+    ####################################################################################################################
+    print("Step 3: get all input offline meters associated with the equipment")
+
+    offline_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_offline_meters m, tbl_equipments_offline_meters em "
+                                 " WHERE m.id = em.offline_meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND em.is_output = false "
+                                 "       AND em.equipment_id = %s ",
+                                 (equipment['id'],))
+        rows_offline_meters = cursor_system_db.fetchall()
+
+        if rows_offline_meters is not None and len(rows_offline_meters) > 0:
+            for row in rows_offline_meters:
+                offline_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 3.1 of equipment_energy_input_category.worker " + str(e)
+        print(error_string)
+        return error_string
+    finally:
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+
+    ####################################################################################################################
+    # stop to the next equipment if this equipment is empty
+    ####################################################################################################################
+    if (meter_list is None or len(meter_list) == 0) and \
+            (virtual_meter_list is None or len(virtual_meter_list) == 0) and \
+            (offline_meter_list is None or len(offline_meter_list) == 0):
+        print("This is an empty equipment ")
+        return None
+
+    ####################################################################################################################
+    # Step 4: determine start datetime and end datetime to aggregate
+    ####################################################################################################################
+    print("Step 4: determine start datetime and end datetime to aggregate")
+    cnx_energy_db = None
+    cursor_energy_db = None
+    try:
+        cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+        cursor_energy_db = cnx_energy_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 4.1 of equipment_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        query = (" SELECT MAX(start_datetime_utc) "
+                 " FROM tbl_equipment_input_category_hourly "
+                 " WHERE equipment_id = %s ")
+        cursor_energy_db.execute(query, (equipment['id'],))
+        row_datetime = cursor_energy_db.fetchone()
+        start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+        start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+        if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+            # replace second and microsecond with 0
+            # note: do not replace minute in case of calculating in half hourly
+            start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+            # start from the next time slot
+            start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+        end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None)
+
+        print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]
+              + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19])
+
+    except Exception as e:
+        error_string = "Error in step 4.2 of equipment_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 5: for each meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_meter_hourly = dict()
+    try:
+        if meter_list is not None and len(meter_list) > 0:
+            for meter in meter_list:
+                meter_id = str(meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_meter_hourly "
+                         " WHERE meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_meter_hourly[meter_id] = None
+                else:
+                    energy_meter_hourly[meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1]
+    except Exception as e:
+        error_string = "Error in step 5.1 of equipment_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 6: for each virtual meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_virtual_meter_hourly = dict()
+    if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+        try:
+            for virtual_meter in virtual_meter_list:
+                virtual_meter_id = str(virtual_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_virtual_meter_hourly "
+                         " WHERE virtual_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_virtual_meter_hourly[virtual_meter_id] = None
+                else:
+                    energy_virtual_meter_hourly[virtual_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1]
+        except Exception as e:
+            error_string = "Error in step 6.1 of equipment_energy_input_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 7: for each offline meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_offline_meter_hourly = dict()
+    if offline_meter_list is not None and len(offline_meter_list) > 0:
+        try:
+            for offline_meter in offline_meter_list:
+                offline_meter_id = str(offline_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_offline_meter_hourly "
+                         " WHERE offline_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_offline_meter_hourly[offline_meter_id] = None
+                else:
+                    energy_offline_meter_hourly[offline_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1]
+
+        except Exception as e:
+            error_string = "Error in step 7.1 of equipment_energy_input_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 8: determine common time slot to aggregate
+    ####################################################################################################################
+
+    common_start_datetime_utc = start_datetime_utc
+    common_end_datetime_utc = end_datetime_utc
+
+    print("Getting common time slot of energy values for all meters")
+    if energy_meter_hourly is not None and len(energy_meter_hourly) > 0:
+        for meter_id, energy_hourly in energy_meter_hourly.items():
+            if energy_hourly is None or len(energy_hourly) == 0:
+                common_start_datetime_utc = None
+                common_end_datetime_utc = None
+                break
+            else:
+                if common_start_datetime_utc < min(energy_hourly.keys()):
+                    common_start_datetime_utc = min(energy_hourly.keys())
+                if common_end_datetime_utc > max(energy_hourly.keys()):
+                    common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all virtual meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_virtual_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all offline meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_offline_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \
+            (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \
+            (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0):
+        # There isn't any energy data
+        print("There isn't any energy data")
+        # continue the for equipment loop to the next equipment
+        print("continue the for equipment loop to the next equipment")
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        return None
+
+    print("common_start_datetime_utc: " + str(common_start_datetime_utc))
+    print("common_end_datetime_utc: " + str(common_end_datetime_utc))
+
+    ####################################################################################################################
+    # Step 9: aggregate energy data in the common time slot by energy categories and hourly
+    ####################################################################################################################
+
+    print("Step 9: aggregate energy data in the common time slot by energy categories and hourly")
+    aggregated_values = list()
+    try:
+        current_datetime_utc = common_start_datetime_utc
+        while common_start_datetime_utc is not None \
+                and common_end_datetime_utc is not None \
+                and current_datetime_utc <= common_end_datetime_utc:
+            aggregated_value = dict()
+            aggregated_value['start_datetime_utc'] = current_datetime_utc
+            aggregated_value['meta_data'] = dict()
+
+            if meter_list is not None and len(meter_list) > 0:
+                for meter in meter_list:
+                    meter_id = str(meter['id'])
+                    energy_category_id = meter['energy_category_id']
+                    actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+                for virtual_meter in virtual_meter_list:
+                    virtual_meter_id = str(virtual_meter['id'])
+                    energy_category_id = virtual_meter['energy_category_id']
+                    actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if offline_meter_list is not None and len(offline_meter_list) > 0:
+                for offline_meter in offline_meter_list:
+                    offline_meter_id = str(offline_meter['id'])
+                    energy_category_id = offline_meter['energy_category_id']
+                    actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            aggregated_values.append(aggregated_value)
+
+            current_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+    except Exception as e:
+        error_string = "Error in step 9 of equipment_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 10: save energy data to energy database
+    ####################################################################################################################
+    print("Step 10: save energy data to energy database")
+
+    if len(aggregated_values) > 0:
+        try:
+            add_values = (" INSERT INTO tbl_equipment_input_category_hourly "
+                          "             (equipment_id, "
+                          "              energy_category_id, "
+                          "              start_datetime_utc, "
+                          "              actual_value) "
+                          " VALUES  ")
+
+            for aggregated_value in aggregated_values:
+                for energy_category_id, actual_value in aggregated_value['meta_data'].items():
+                    add_values += " (" + str(equipment['id']) + ","
+                    add_values += " " + str(energy_category_id) + ","
+                    add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "',"
+                    add_values += str(actual_value) + "), "
+            print("add_values:" + add_values)
+            # trim ", " at the end of string and then execute
+            cursor_energy_db.execute(add_values[:-2])
+            cnx_energy_db.commit()
+
+        except Exception as e:
+            error_string = "Error in step 10.1 of equipment_energy_input_category.worker " + str(e)
+            print(error_string)
+            return error_string
+        finally:
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+    else:
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()

myems-aggregation/equipment_energy_input_item.py

@@ -0,0 +1,527 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+from multiprocessing import Pool
+import random
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all equipments
+# Step 2: Create multiprocessing pool to call worker in parallel
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all equipments
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of equipment_energy_input_item.main " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name "
+                                     " FROM tbl_equipments "
+                                     " ORDER BY id ")
+            rows_equipments = cursor_system_db.fetchall()
+
+            if rows_equipments is None or len(rows_equipments) == 0:
+                print("There isn't any equipments ")
+                # sleep and continue the outer loop to reconnect the database
+                time.sleep(60)
+                continue
+
+            equipment_list = list()
+            for row in rows_equipments:
+                equipment_list.append({"id": row[0], "name": row[1]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of equipment_energy_input_item.main " + str(e))
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        finally:
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+
+        print("Got all equipments in MyEMS System Database")
+
+        # shuffle the equipment list for randomly calculating the meter hourly value
+        random.shuffle(equipment_list)
+
+        ################################################################################################################
+        # Step 2: Create multiprocessing pool to call worker in parallel
+        ################################################################################################################
+        p = Pool(processes=config.pool_size)
+        error_list = p.map(worker, equipment_list)
+        p.close()
+        p.join()
+
+        for error in error_list:
+            if error is not None and len(error) > 0:
+                logger.error(error)
+
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of outer while
+
+
+########################################################################################################################
+# PROCEDURES:
+#   Step 1: get all input meters associated with the equipment
+#   Step 2: get all input virtual meters associated with the equipment
+#   Step 3: get all input offline meters associated with the equipment
+#   Step 4: determine start datetime and end datetime to aggregate
+#   Step 5: for each meter in list, get energy input data from energy database
+#   Step 6: for each virtual meter in list, get energy input data from energy database
+#   Step 7: for each offline meter in list, get energy input data from energy database
+#   Step 8: determine common time slot to aggregate
+#   Step 9: aggregate energy data in the common time slot by energy items and hourly
+#   Step 10: save energy data to energy database
+#
+# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter
+########################################################################################################################
+
+def worker(equipment):
+    ####################################################################################################################
+    # Step 1: get all input meters associated with the equipment
+    ####################################################################################################################
+    print("Step 1: get all input meters associated with the equipment " + str(equipment['name']))
+
+    meter_list = list()
+    cnx_system_db = None
+    cursor_system_db = None
+    try:
+        cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+        cursor_system_db = cnx_system_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 1.1 of equipment_energy_input_item.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id "
+                                 " FROM tbl_meters m, tbl_equipments_meters em "
+                                 " WHERE m.id = em.meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND m.energy_item_id is NOT NULL "
+                                 "       AND em.is_output = false "
+                                 "       AND em.equipment_id = %s ",
+                                 (equipment['id'],))
+        rows_meters = cursor_system_db.fetchall()
+
+        if rows_meters is not None and len(rows_meters) > 0:
+            for row in rows_meters:
+                meter_list.append({"id": row[0],
+                                   "name": row[1],
+                                   "energy_item_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 1.2 of equipment_energy_input_item.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 2: get all input virtual meters associated with the equipment
+    ####################################################################################################################
+    print("Step 2: get all input virtual meters associated with the equipment")
+    virtual_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id "
+                                 " FROM tbl_virtual_meters m, tbl_equipments_virtual_meters em "
+                                 " WHERE m.id = em.virtual_meter_id "
+                                 "       AND m.energy_item_id is NOT NULL "
+                                 "       AND m.is_counted = true "
+                                 "       AND em.is_output = false "
+                                 "       AND em.equipment_id = %s ",
+                                 (equipment['id'],))
+        rows_virtual_meters = cursor_system_db.fetchall()
+
+        if rows_virtual_meters is not None and len(rows_virtual_meters) > 0:
+            for row in rows_virtual_meters:
+                virtual_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_item_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 2.1 of equipment_energy_input_item.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 3: get all input offline meters associated with the equipment
+    ####################################################################################################################
+    print("Step 3: get all input offline meters associated with the equipment")
+
+    offline_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id "
+                                 " FROM tbl_offline_meters m, tbl_equipments_offline_meters em "
+                                 " WHERE m.id = em.offline_meter_id "
+                                 "       AND m.energy_item_id is NOT NULL "
+                                 "       AND m.is_counted = true "
+                                 "       AND em.is_output = false "
+                                 "       AND em.equipment_id = %s ",
+                                 (equipment['id'],))
+        rows_offline_meters = cursor_system_db.fetchall()
+
+        if rows_offline_meters is not None and len(rows_offline_meters) > 0:
+            for row in rows_offline_meters:
+                offline_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_item_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 3.1 of equipment_energy_input_item.worker " + str(e)
+        print(error_string)
+        return error_string
+    finally:
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+
+    ####################################################################################################################
+    # stop to the next equipment if this equipment is empty
+    ####################################################################################################################
+    if (meter_list is None or len(meter_list) == 0) and \
+            (virtual_meter_list is None or len(virtual_meter_list) == 0) and \
+            (offline_meter_list is None or len(offline_meter_list) == 0):
+        print("This is an empty equipment ")
+        return None
+
+    ####################################################################################################################
+    # Step 4: determine start datetime and end datetime to aggregate
+    ####################################################################################################################
+    print("Step 4: determine start datetime and end datetime to aggregate")
+    cnx_energy_db = None
+    cursor_energy_db = None
+    try:
+        cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+        cursor_energy_db = cnx_energy_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 4.1 of equipment_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        query = (" SELECT MAX(start_datetime_utc) "
+                 " FROM tbl_equipment_input_item_hourly "
+                 " WHERE equipment_id = %s ")
+        cursor_energy_db.execute(query, (equipment['id'],))
+        row_datetime = cursor_energy_db.fetchone()
+        start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+        start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+        if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+            # replace second and microsecond with 0
+            # note: do not replace minute in case of calculating in half hourly
+            start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+            # start from the next time slot
+            start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+        end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None)
+
+        print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]
+              + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19])
+
+    except Exception as e:
+        error_string = "Error in step 4.2 of equipment_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 5: for each meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_meter_hourly = dict()
+    try:
+        if meter_list is not None and len(meter_list) > 0:
+            for meter in meter_list:
+                meter_id = str(meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_meter_hourly "
+                         " WHERE meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_meter_hourly[meter_id] = None
+                else:
+                    energy_meter_hourly[meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1]
+    except Exception as e:
+        error_string = "Error in step 5.1 of equipment_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 6: for each virtual meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_virtual_meter_hourly = dict()
+    if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+        try:
+            for virtual_meter in virtual_meter_list:
+                virtual_meter_id = str(virtual_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_virtual_meter_hourly "
+                         " WHERE virtual_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_virtual_meter_hourly[virtual_meter_id] = None
+                else:
+                    energy_virtual_meter_hourly[virtual_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1]
+        except Exception as e:
+            error_string = "Error in step 6.1 of equipment_energy_input_item.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 7: for each offline meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_offline_meter_hourly = dict()
+    if offline_meter_list is not None and len(offline_meter_list) > 0:
+        try:
+            for offline_meter in offline_meter_list:
+                offline_meter_id = str(offline_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_offline_meter_hourly "
+                         " WHERE offline_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_offline_meter_hourly[offline_meter_id] = None
+                else:
+                    energy_offline_meter_hourly[offline_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1]
+
+        except Exception as e:
+            error_string = "Error in step 7.1 of equipment_energy_input_item.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 8: determine common time slot to aggregate
+    ####################################################################################################################
+
+    common_start_datetime_utc = start_datetime_utc
+    common_end_datetime_utc = end_datetime_utc
+
+    print("Getting common time slot of energy values for all meters")
+    if energy_meter_hourly is not None and len(energy_meter_hourly) > 0:
+        for meter_id, energy_hourly in energy_meter_hourly.items():
+            if energy_hourly is None or len(energy_hourly) == 0:
+                common_start_datetime_utc = None
+                common_end_datetime_utc = None
+                break
+            else:
+                if common_start_datetime_utc < min(energy_hourly.keys()):
+                    common_start_datetime_utc = min(energy_hourly.keys())
+                if common_end_datetime_utc > max(energy_hourly.keys()):
+                    common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all virtual meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_virtual_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all offline meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_offline_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \
+            (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \
+            (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0):
+        # There isn't any energy data
+        print("There isn't any energy data")
+        # continue the for equipment loop to the next equipment
+        print("continue the for equipment loop to the next equipment")
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        return None
+
+    print("common_start_datetime_utc: " + str(common_start_datetime_utc))
+    print("common_end_datetime_utc: " + str(common_end_datetime_utc))
+
+    ####################################################################################################################
+    # Step 9: aggregate energy data in the common time slot by energy items and hourly
+    ####################################################################################################################
+
+    print("Step 9: aggregate energy data in the common time slot by energy items and hourly")
+    aggregated_values = list()
+    try:
+        current_datetime_utc = common_start_datetime_utc
+        while common_start_datetime_utc is not None \
+                and common_end_datetime_utc is not None \
+                and current_datetime_utc <= common_end_datetime_utc:
+            aggregated_value = dict()
+            aggregated_value['start_datetime_utc'] = current_datetime_utc
+            aggregated_value['meta_data'] = dict()
+
+            if meter_list is not None and len(meter_list) > 0:
+                for meter in meter_list:
+                    meter_id = str(meter['id'])
+                    energy_item_id = meter['energy_item_id']
+                    actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_item_id] = \
+                        aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value
+
+            if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+                for virtual_meter in virtual_meter_list:
+                    virtual_meter_id = str(virtual_meter['id'])
+                    energy_item_id = virtual_meter['energy_item_id']
+                    actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_item_id] = \
+                        aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value
+
+            if offline_meter_list is not None and len(offline_meter_list) > 0:
+                for offline_meter in offline_meter_list:
+                    offline_meter_id = str(offline_meter['id'])
+                    energy_item_id = offline_meter['energy_item_id']
+                    actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_item_id] = \
+                        aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value
+
+            aggregated_values.append(aggregated_value)
+
+            current_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+    except Exception as e:
+        error_string = "Error in step 9 of equipment_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 10: save energy data to energy database
+    ####################################################################################################################
+    print("Step 10: save energy data to energy database")
+
+    if len(aggregated_values) > 0:
+        try:
+            add_values = (" INSERT INTO tbl_equipment_input_item_hourly "
+                          "             (equipment_id, "
+                          "              energy_item_id, "
+                          "              start_datetime_utc, "
+                          "              actual_value) "
+                          " VALUES  ")
+
+            for aggregated_value in aggregated_values:
+                for energy_item_id, actual_value in aggregated_value['meta_data'].items():
+                    add_values += " (" + str(equipment['id']) + ","
+                    add_values += " " + str(energy_item_id) + ","
+                    add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "',"
+                    add_values += str(actual_value) + "), "
+            print("add_values:" + add_values)
+            # trim ", " at the end of string and then execute
+            cursor_energy_db.execute(add_values[:-2])
+            cnx_energy_db.commit()
+
+        except Exception as e:
+            error_string = "Error in step 10.1 of equipment_energy_input_item.worker " + str(e)
+            print(error_string)
+            return error_string
+        finally:
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+    else:
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()

myems-aggregation/equipment_energy_output_category.py

@@ -0,0 +1,524 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+from multiprocessing import Pool
+import random
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all equipments
+# Step 2: Create multiprocessing pool to call worker in parallel
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all equipments
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of equipment_energy_output_category.main " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name "
+                                     " FROM tbl_equipments "
+                                     " ORDER BY id ")
+            rows_equipments = cursor_system_db.fetchall()
+
+            if rows_equipments is None or len(rows_equipments) == 0:
+                print("There isn't any equipments ")
+                # sleep and continue the outer loop to reconnect the database
+                time.sleep(60)
+                continue
+
+            equipment_list = list()
+            for row in rows_equipments:
+                equipment_list.append({"id": row[0], "name": row[1]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of equipment_energy_output_category.main " + str(e))
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        finally:
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+
+        print("Got all equipments in MyEMS System Database")
+
+        # shuffle the equipment list for randomly calculating the meter hourly value
+        random.shuffle(equipment_list)
+
+        ################################################################################################################
+        # Step 2: Create multiprocessing pool to call worker in parallel
+        ################################################################################################################
+        p = Pool(processes=config.pool_size)
+        error_list = p.map(worker, equipment_list)
+        p.close()
+        p.join()
+
+        for error in error_list:
+            if error is not None and len(error) > 0:
+                logger.error(error)
+
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of outer while
+
+
+########################################################################################################################
+# PROCEDURES:
+#   Step 1: get all output meters associated with the equipment
+#   Step 2: get all output virtual meters associated with the equipment
+#   Step 3: get all output offline meters associated with the equipment
+#   Step 4: determine start datetime and end datetime to aggregate
+#   Step 5: for each meter in list, get energy output data from energy database
+#   Step 6: for each virtual meter in list, get energy output data from energy database
+#   Step 7: for each offline meter in list, get energy output data from energy database
+#   Step 8: determine common time slot to aggregate
+#   Step 9: aggregate energy data in the common time slot by energy categories and hourly
+#   Step 10: save energy data to energy database
+#
+# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter
+########################################################################################################################
+
+def worker(equipment):
+    ####################################################################################################################
+    # Step 1: get all output meters associated with the equipment
+    ####################################################################################################################
+    print("Step 1: get all output meters associated with the equipment " + str(equipment['name']))
+
+    meter_list = list()
+    cnx_system_db = None
+    cursor_system_db = None
+    try:
+        cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+        cursor_system_db = cnx_system_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 1.1 of equipment_energy_output_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_meters m, tbl_equipments_meters em "
+                                 " WHERE m.id = em.meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND em.is_output = true "
+                                 "       AND em.equipment_id = %s ",
+                                 (equipment['id'],))
+        rows_meters = cursor_system_db.fetchall()
+
+        if rows_meters is not None and len(rows_meters) > 0:
+            for row in rows_meters:
+                meter_list.append({"id": row[0],
+                                   "name": row[1],
+                                   "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 1.2 of equipment_energy_output_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 2: get all output virtual meters associated with the equipment
+    ####################################################################################################################
+    print("Step 2: get all output virtual meters associated with the equipment")
+    virtual_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_virtual_meters m, tbl_equipments_virtual_meters em "
+                                 " WHERE m.id = em.virtual_meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND em.is_output = true "
+                                 "       AND em.equipment_id = %s ",
+                                 (equipment['id'],))
+        rows_virtual_meters = cursor_system_db.fetchall()
+
+        if rows_virtual_meters is not None and len(rows_virtual_meters) > 0:
+            for row in rows_virtual_meters:
+                virtual_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 2.1 of equipment_energy_output_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 3: get all output offline meters associated with the equipment
+    ####################################################################################################################
+    print("Step 3: get all output offline meters associated with the equipment")
+
+    offline_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_offline_meters m, tbl_equipments_offline_meters em "
+                                 " WHERE m.id = em.offline_meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND em.is_output = true "
+                                 "       AND em.equipment_id = %s ",
+                                 (equipment['id'],))
+        rows_offline_meters = cursor_system_db.fetchall()
+
+        if rows_offline_meters is not None and len(rows_offline_meters) > 0:
+            for row in rows_offline_meters:
+                offline_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 3.1 of equipment_energy_output_category.worker " + str(e)
+        print(error_string)
+        return error_string
+    finally:
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+
+    ####################################################################################################################
+    # stop to the next equipment if this equipment is empty
+    ####################################################################################################################
+    if (meter_list is None or len(meter_list) == 0) and \
+            (virtual_meter_list is None or len(virtual_meter_list) == 0) and \
+            (offline_meter_list is None or len(offline_meter_list) == 0):
+        print("This is an empty equipment ")
+        return None
+
+    ####################################################################################################################
+    # Step 4: determine start datetime and end datetime to aggregate
+    ####################################################################################################################
+    print("Step 4: determine start datetime and end datetime to aggregate")
+    cnx_energy_db = None
+    cursor_energy_db = None
+    try:
+        cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+        cursor_energy_db = cnx_energy_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 4.1 of equipment_energy_output_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        query = (" SELECT MAX(start_datetime_utc) "
+                 " FROM tbl_equipment_output_category_hourly "
+                 " WHERE equipment_id = %s ")
+        cursor_energy_db.execute(query, (equipment['id'],))
+        row_datetime = cursor_energy_db.fetchone()
+        start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+        start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+        if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+            # replace second and microsecond with 0
+            # note: do not replace minute in case of calculating in half hourly
+            start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+            # start from the next time slot
+            start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+        end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None)
+
+        print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]
+              + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19])
+
+    except Exception as e:
+        error_string = "Error in step 4.2 of equipment_energy_output_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 5: for each meter in list, get energy output data from energy database
+    ####################################################################################################################
+    energy_meter_hourly = dict()
+    try:
+        if meter_list is not None and len(meter_list) > 0:
+            for meter in meter_list:
+                meter_id = str(meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_meter_hourly "
+                         " WHERE meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_meter_hourly[meter_id] = None
+                else:
+                    energy_meter_hourly[meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1]
+    except Exception as e:
+        error_string = "Error in step 5.1 of equipment_energy_output_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 6: for each virtual meter in list, get energy output data from energy database
+    ####################################################################################################################
+    energy_virtual_meter_hourly = dict()
+    if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+        try:
+            for virtual_meter in virtual_meter_list:
+                virtual_meter_id = str(virtual_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_virtual_meter_hourly "
+                         " WHERE virtual_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_virtual_meter_hourly[virtual_meter_id] = None
+                else:
+                    energy_virtual_meter_hourly[virtual_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1]
+        except Exception as e:
+            error_string = "Error in step 6.1 of equipment_energy_output_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 7: for each offline meter in list, get energy output data from energy database
+    ####################################################################################################################
+    energy_offline_meter_hourly = dict()
+    if offline_meter_list is not None and len(offline_meter_list) > 0:
+        try:
+            for offline_meter in offline_meter_list:
+                offline_meter_id = str(offline_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_offline_meter_hourly "
+                         " WHERE offline_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_offline_meter_hourly[offline_meter_id] = None
+                else:
+                    energy_offline_meter_hourly[offline_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1]
+
+        except Exception as e:
+            error_string = "Error in step 7.1 of equipment_energy_output_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 8: determine common time slot to aggregate
+    ####################################################################################################################
+
+    common_start_datetime_utc = start_datetime_utc
+    common_end_datetime_utc = end_datetime_utc
+
+    print("Getting common time slot of energy values for all meters")
+    if energy_meter_hourly is not None and len(energy_meter_hourly) > 0:
+        for meter_id, energy_hourly in energy_meter_hourly.items():
+            if energy_hourly is None or len(energy_hourly) == 0:
+                common_start_datetime_utc = None
+                common_end_datetime_utc = None
+                break
+            else:
+                if common_start_datetime_utc < min(energy_hourly.keys()):
+                    common_start_datetime_utc = min(energy_hourly.keys())
+                if common_end_datetime_utc > max(energy_hourly.keys()):
+                    common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all virtual meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_virtual_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all offline meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_offline_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \
+            (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \
+            (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0):
+        # There isn't any energy data
+        print("There isn't any energy data")
+        # continue the for equipment loop to the next equipment
+        print("continue the for equipment loop to the next equipment")
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        return None
+
+    print("common_start_datetime_utc: " + str(common_start_datetime_utc))
+    print("common_end_datetime_utc: " + str(common_end_datetime_utc))
+
+    ####################################################################################################################
+    # Step 9: aggregate energy data in the common time slot by energy categories and hourly
+    ####################################################################################################################
+
+    print("Step 9: aggregate energy data in the common time slot by energy categories and hourly")
+    aggregated_values = list()
+    try:
+        current_datetime_utc = common_start_datetime_utc
+        while common_start_datetime_utc is not None \
+                and common_end_datetime_utc is not None \
+                and current_datetime_utc <= common_end_datetime_utc:
+            aggregated_value = dict()
+            aggregated_value['start_datetime_utc'] = current_datetime_utc
+            aggregated_value['meta_data'] = dict()
+
+            if meter_list is not None and len(meter_list) > 0:
+                for meter in meter_list:
+                    meter_id = str(meter['id'])
+                    energy_category_id = meter['energy_category_id']
+                    actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+                for virtual_meter in virtual_meter_list:
+                    virtual_meter_id = str(virtual_meter['id'])
+                    energy_category_id = virtual_meter['energy_category_id']
+                    actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if offline_meter_list is not None and len(offline_meter_list) > 0:
+                for offline_meter in offline_meter_list:
+                    offline_meter_id = str(offline_meter['id'])
+                    energy_category_id = offline_meter['energy_category_id']
+                    actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            aggregated_values.append(aggregated_value)
+
+            current_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+    except Exception as e:
+        error_string = "Error in step 9 of equipment_energy_output_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 10: save energy data to energy database
+    ####################################################################################################################
+    print("Step 10: save energy data to energy database")
+
+    if len(aggregated_values) > 0:
+        try:
+            add_values = (" INSERT INTO tbl_equipment_output_category_hourly "
+                          "             (equipment_id, "
+                          "              energy_category_id, "
+                          "              start_datetime_utc, "
+                          "              actual_value) "
+                          " VALUES  ")
+
+            for aggregated_value in aggregated_values:
+                for energy_category_id, actual_value in aggregated_value['meta_data'].items():
+                    add_values += " (" + str(equipment['id']) + ","
+                    add_values += " " + str(energy_category_id) + ","
+                    add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "',"
+                    add_values += str(actual_value) + "), "
+            print("add_values:" + add_values)
+            # trim ", " at the end of string and then execute
+            cursor_energy_db.execute(add_values[:-2])
+            cnx_energy_db.commit()
+
+        except Exception as e:
+            error_string = "Error in step 10.1 of equipment_energy_output_category.worker " + str(e)
+            print(error_string)
+            return error_string
+        finally:
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+    else:
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()

myems-aggregation/main.py

@@ -0,0 +1,147 @@
+import logging
+from logging.handlers import RotatingFileHandler
+from multiprocessing import Process
+
+
+import combined_equipment_energy_input_category
+import combined_equipment_energy_input_item
+import combined_equipment_energy_output_category
+
+import combined_equipment_billing_input_category
+import combined_equipment_billing_input_item
+import combined_equipment_billing_output_category
+
+import equipment_energy_input_category
+import equipment_energy_input_item
+import equipment_energy_output_category
+
+import equipment_billing_input_category
+import equipment_billing_input_item
+import equipment_billing_output_category
+
+import meter_billing
+
+import shopfloor_billing_input_category
+import shopfloor_billing_input_item
+
+import shopfloor_energy_input_category
+import shopfloor_energy_input_item
+
+import space_billing_input_category
+import space_billing_input_item
+import space_billing_output_category
+
+import space_energy_input_category
+import space_energy_input_item
+import space_energy_output_category
+
+import store_billing_input_category
+import store_billing_input_item
+
+import store_energy_input_category
+import store_energy_input_item
+
+import tenant_billing_input_category
+import tenant_billing_input_item
+
+import tenant_energy_input_category
+import tenant_energy_input_item
+
+
+def main():
+    """main"""
+    # create logger
+    logger = logging.getLogger('myems-aggregation')
+    # specifies the lowest-severity log message a logger will handle,
+    # where debug is the lowest built-in severity level and critical is the highest built-in severity.
+    # For example, if the severity level is INFO, the logger will handle only INFO, WARNING, ERROR, and CRITICAL
+    # messages and will ignore DEBUG messages.
+    logger.setLevel(logging.ERROR)
+    # create file handler which logs messages
+    fh = RotatingFileHandler('myems-aggregation.log', maxBytes=1024*1024, backupCount=1)
+    # create formatter and add it to the handlers
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    fh.setFormatter(formatter)
+    # add the handlers to logger
+    logger.addHandler(fh)
+
+    # combined equipment energy input by energy categories
+    Process(target=combined_equipment_energy_input_category.main, args=(logger,)).start()
+    # combined equipment energy input by energy items
+    Process(target=combined_equipment_energy_input_item.main, args=(logger,)).start()
+    # combined equipment energy output by energy categories
+    Process(target=combined_equipment_energy_output_category.main, args=(logger,)).start()
+
+    # combined equipment billing input by energy categories
+    Process(target=combined_equipment_billing_input_category.main, args=(logger,)).start()
+    # combined equipment billing input by energy items
+    Process(target=combined_equipment_billing_input_item.main, args=(logger,)).start()
+    # combined equipment billing output by energy categories
+    Process(target=combined_equipment_billing_output_category.main, args=(logger,)).start()
+
+    # equipment billing input by energy categories
+    Process(target=equipment_billing_input_category.main, args=(logger,)).start()
+    # equipment billing input by energy items
+    Process(target=equipment_billing_input_item.main, args=(logger,)).start()
+    # equipment billing output by energy categories
+    Process(target=equipment_billing_output_category.main, args=(logger,)).start()
+
+    # equipment energy input by energy categories
+    Process(target=equipment_energy_input_category.main, args=(logger,)).start()
+    # equipment energy input by energy items
+    Process(target=equipment_energy_input_item.main, args=(logger,)).start()
+    # equipment energy output by energy categories
+    Process(target=equipment_energy_output_category.main, args=(logger,)).start()
+
+    # meter billing
+    Process(target=meter_billing.main, args=(logger,)).start()
+
+    # shopfloor billing input by energy categories
+    Process(target=shopfloor_billing_input_category.main, args=(logger,)).start()
+    # shopfloor billing input by energy items
+    Process(target=shopfloor_billing_input_item.main, args=(logger,)).start()
+
+    # shopfloor energy input by energy categories
+    Process(target=shopfloor_energy_input_category.main, args=(logger,)).start()
+    # shopfloor energy input by energy items
+    Process(target=shopfloor_energy_input_item.main, args=(logger,)).start()
+
+    # space billing input by energy categories
+    Process(target=space_billing_input_category.main, args=(logger,)).start()
+    # space billing input by energy items
+    Process(target=space_billing_input_item.main, args=(logger,)).start()
+    # space billing output by energy categories
+    Process(target=space_billing_output_category.main, args=(logger,)).start()
+
+    # space energy input by energy categories
+    Process(target=space_energy_input_category.main, args=(logger,)).start()
+    # space energy input by energy items
+    Process(target=space_energy_input_item.main, args=(logger,)).start()
+    # space energy output by energy categories
+    Process(target=space_energy_output_category.main, args=(logger,)).start()
+
+    # store billing input by energy categories
+    Process(target=store_billing_input_category.main, args=(logger,)).start()
+    # store billing input by energy items
+    Process(target=store_billing_input_item.main, args=(logger,)).start()
+
+    # store energy input by energy categories
+    Process(target=store_energy_input_category.main, args=(logger,)).start()
+    # store energy input by energy items
+    Process(target=store_energy_input_item.main, args=(logger,)).start()
+
+    # tenant billing input by energy categories
+    Process(target=tenant_billing_input_category.main, args=(logger,)).start()
+    # tenant billing input by energy items
+    Process(target=tenant_billing_input_item.main, args=(logger,)).start()
+
+    # tenant energy input by energy categories
+    Process(target=tenant_energy_input_category.main, args=(logger,)).start()
+    # tenant energy input by energy items
+    Process(target=tenant_energy_input_item.main, args=(logger,)).start()
+
+
+if __name__ == '__main__':
+    main()
+
+

myems-aggregation/meter_billing.py

@@ -0,0 +1,266 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+import tariff
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all meters
+# for each meter in list:
+#   Step 2: get the latest start_datetime_utc
+#   Step 3: get all energy data since the latest start_datetime_utc
+#   Step 4: get tariffs
+#   Step 5: calculate billing by multiplying energy with tariff
+#   Step 6: save billing data to database
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all meters
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of meter_billing " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name, energy_category_id, cost_center_id "
+                                     " FROM tbl_meters "
+                                     " ORDER BY id ")
+            rows_meters = cursor_system_db.fetchall()
+
+            if rows_meters is None or len(rows_meters) == 0:
+                print("Step 1.2: There isn't any equipments. ")
+                if cursor_system_db:
+                    cursor_system_db.close()
+                if cnx_system_db:
+                    cnx_system_db.close()
+                # sleep and continue the outermost while loop
+                time.sleep(60)
+                continue
+
+            meter_list = list()
+            for row in rows_meters:
+                meter_list.append({"id": row[0],
+                                   "name": row[1],
+                                   "energy_category_id": row[2],
+                                   "cost_center_id": row[3]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of meter_billing " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Step 1.2: Got all meters from MyEMS System Database")
+
+        cnx_energy_db = None
+        cursor_energy_db = None
+        try:
+            cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+            cursor_energy_db = cnx_energy_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.3 of meter_billing " + str(e))
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Energy Database")
+
+        cnx_billing_db = None
+        cursor_billing_db = None
+        try:
+            cnx_billing_db = mysql.connector.connect(**config.myems_billing_db)
+            cursor_billing_db = cnx_billing_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.4 of meter_billing " + str(e))
+            if cursor_billing_db:
+                cursor_billing_db.close()
+            if cnx_billing_db:
+                cnx_billing_db.close()
+
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Billing Database")
+
+        for meter in meter_list:
+
+            ############################################################################################################
+            # Step 2: get the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 2: get the latest start_datetime_utc from billing database for " + meter['name'])
+            try:
+                cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) "
+                                          " FROM tbl_meter_hourly "
+                                          " WHERE meter_id = %s ",
+                                          (meter['id'], ))
+                row_datetime = cursor_billing_db.fetchone()
+                start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+                start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+                if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+                    # replace second and microsecond with 0
+                    # note: do not replace minute in case of calculating in half hourly
+                    start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+                    # start from the next time slot
+                    start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+                print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19])
+            except Exception as e:
+                logger.error("Error in step 2 of meter_billing " + str(e))
+                # break the for meter loop
+                break
+
+            ############################################################################################################
+            # Step 3: get all energy data since the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 3: get all energy data since the latest start_datetime_utc")
+
+            query = (" SELECT start_datetime_utc, actual_value "
+                     " FROM tbl_meter_hourly "
+                     " WHERE meter_id = %s AND start_datetime_utc >= %s "
+                     " ORDER BY id ")
+            cursor_energy_db.execute(query, (meter['id'], start_datetime_utc, ))
+            rows_hourly = cursor_energy_db.fetchall()
+
+            if rows_hourly is None or len(rows_hourly) == 0:
+                print("Step 3: There isn't any energy input data to calculate. ")
+                # continue the for meter loop
+                continue
+
+            energy_dict = dict()
+            energy_category_list = list()
+            energy_category_list.append(meter['energy_category_id'])
+            end_datetime_utc = start_datetime_utc
+            for row_hourly in rows_hourly:
+                current_datetime_utc = row_hourly[0]
+                actual_value = row_hourly[1]
+                if energy_dict.get(current_datetime_utc) is None:
+                    energy_dict[current_datetime_utc] = dict()
+                energy_dict[current_datetime_utc][meter['energy_category_id']] = actual_value
+                if current_datetime_utc > end_datetime_utc:
+                    end_datetime_utc = current_datetime_utc
+
+            ############################################################################################################
+            # Step 4: get tariffs
+            ############################################################################################################
+            print("Step 4: get tariffs")
+            tariff_dict = dict()
+            for energy_category_id in energy_category_list:
+                tariff_dict[energy_category_id] = tariff.get_energy_category_tariffs(meter['cost_center_id'],
+                                                                                     energy_category_id,
+                                                                                     start_datetime_utc,
+                                                                                     end_datetime_utc)
+            ############################################################################################################
+            # Step 5: calculate billing by multiplying energy with tariff
+            ############################################################################################################
+            print("Step 5: calculate billing by multiplying energy with tariff")
+            billing_dict = dict()
+
+            if len(energy_dict) > 0:
+                for current_datetime_utc in energy_dict.keys():
+                    billing_dict[current_datetime_utc] = dict()
+                    for energy_category_id in energy_category_list:
+                        current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc)
+                        current_energy = energy_dict[current_datetime_utc].get(energy_category_id)
+                        if current_tariff is not None \
+                                and isinstance(current_tariff, Decimal) \
+                                and current_energy is not None \
+                                and isinstance(current_energy, Decimal):
+                            billing_dict[current_datetime_utc][energy_category_id] = \
+                                current_energy * current_tariff
+
+                    if len(billing_dict[current_datetime_utc]) == 0:
+                        del billing_dict[current_datetime_utc]
+
+            ############################################################################################################
+            # Step 6: save billing data to billing database
+            ############################################################################################################
+            print("Step 6: save billing data to billing database")
+
+            if len(billing_dict) > 0:
+                try:
+                    add_values = (" INSERT INTO tbl_meter_hourly "
+                                  "             (meter_id, "
+                                  "              start_datetime_utc, "
+                                  "              actual_value) "
+                                  " VALUES  ")
+
+                    for current_datetime_utc in billing_dict:
+                        for energy_category_id in energy_category_list:
+                            current_billing = billing_dict[current_datetime_utc].get(energy_category_id)
+                            if current_billing is not None and isinstance(current_billing, Decimal):
+                                add_values += " (" + str(meter['id']) + ","
+                                add_values += "'" + current_datetime_utc.isoformat()[0:19] + "',"
+                                add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), "
+                    print("add_values:" + add_values)
+                    # trim ", " at the end of string and then execute
+                    cursor_billing_db.execute(add_values[:-2])
+                    cnx_billing_db.commit()
+                except Exception as e:
+                    logger.error("Error in step 6 of meter_billing " + str(e))
+                    # break the for meter loop
+                    break
+
+        # end of for meter loop
+        if cnx_system_db:
+            cnx_system_db.close()
+        if cursor_system_db:
+            cursor_system_db.close()
+
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        if cursor_energy_db:
+            cursor_energy_db.close()
+
+        if cnx_billing_db:
+            cnx_billing_db.close()
+        if cursor_billing_db:
+            cursor_billing_db.close()
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of the outermost while loop

myems-aggregation/myems-aggregation.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=myems-aggregation daemon
+After=network.target
+
+[Service]
+User=root
+Group=root
+ExecStart=/usr/bin/python3 /myems-aggregation/main.py
+ExecReload=/bin/kill -s HUP $MAINPID
+ExecStop=/bin/kill -s TERM $MAINPID
+PrivateTmp=true
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

myems-aggregation/shopfloor_billing_input_category.py

@@ -0,0 +1,269 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+import tariff
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all shopfloors
+# for each shopfloor in list:
+#   Step 2: get the latest start_datetime_utc
+#   Step 3: get all energy input data since the latest start_datetime_utc
+#   Step 4: get tariffs
+#   Step 5: calculate billing by multiplying energy with tariff
+#   Step 6: save billing data to database
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all shopfloors
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of shopfloor_billing_input_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name, cost_center_id "
+                                     " FROM tbl_shopfloors "
+                                     " ORDER BY id ")
+            rows_shopfloors = cursor_system_db.fetchall()
+
+            if rows_shopfloors is None or len(rows_shopfloors) == 0:
+                print("Step 1.2: There isn't any shopfloors. ")
+                if cursor_system_db:
+                    cursor_system_db.close()
+                if cnx_system_db:
+                    cnx_system_db.close()
+                # sleep and continue the outermost while loop
+                time.sleep(60)
+                continue
+
+            shopfloor_list = list()
+            for row in rows_shopfloors:
+                shopfloor_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of shopfloor_billing_input_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Step 1.2: Got all shopfloors from MyEMS System Database")
+
+        cnx_energy_db = None
+        cursor_energy_db = None
+        try:
+            cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+            cursor_energy_db = cnx_energy_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.3 of shopfloor_billing_input_category " + str(e))
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Energy Database")
+
+        cnx_billing_db = None
+        cursor_billing_db = None
+        try:
+            cnx_billing_db = mysql.connector.connect(**config.myems_billing_db)
+            cursor_billing_db = cnx_billing_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.4 of shopfloor_billing_input_category " + str(e))
+            if cursor_billing_db:
+                cursor_billing_db.close()
+            if cnx_billing_db:
+                cnx_billing_db.close()
+
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Billing Database")
+
+        for shopfloor in shopfloor_list:
+
+            ############################################################################################################
+            # Step 2: get the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 2: get the latest start_datetime_utc from billing database for " + shopfloor['name'])
+            try:
+                cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) "
+                                          " FROM tbl_shopfloor_input_category_hourly "
+                                          " WHERE shopfloor_id = %s ",
+                                          (shopfloor['id'], ))
+                row_datetime = cursor_billing_db.fetchone()
+                start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+                start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+                if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+                    # replace second and microsecond with 0
+                    # note: do not replace minute in case of calculating in half hourly
+                    start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+                    # start from the next time slot
+                    start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+                print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19])
+            except Exception as e:
+                logger.error("Error in step 2 of shopfloor_billing_input_category " + str(e))
+                # break the for shopfloor loop
+                break
+
+            ############################################################################################################
+            # Step 3: get all energy input data since the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 3: get all energy input data since the latest start_datetime_utc")
+
+            query = (" SELECT start_datetime_utc, energy_category_id, actual_value "
+                     " FROM tbl_shopfloor_input_category_hourly "
+                     " WHERE shopfloor_id = %s AND start_datetime_utc >= %s "
+                     " ORDER BY id ")
+            cursor_energy_db.execute(query, (shopfloor['id'], start_datetime_utc, ))
+            rows_hourly = cursor_energy_db.fetchall()
+
+            if rows_hourly is None or len(rows_hourly) == 0:
+                print("Step 3: There isn't any energy input data to calculate. ")
+                # continue the for shopfloor loop
+                continue
+
+            energy_dict = dict()
+            energy_category_list = list()
+            end_datetime_utc = start_datetime_utc
+            for row_hourly in rows_hourly:
+                current_datetime_utc = row_hourly[0]
+                energy_category_id = row_hourly[1]
+
+                if energy_category_id not in energy_category_list:
+                    energy_category_list.append(energy_category_id)
+
+                actual_value = row_hourly[2]
+                if energy_dict.get(current_datetime_utc) is None:
+                    energy_dict[current_datetime_utc] = dict()
+                energy_dict[current_datetime_utc][energy_category_id] = actual_value
+                if current_datetime_utc > end_datetime_utc:
+                    end_datetime_utc = current_datetime_utc
+
+            ############################################################################################################
+            # Step 4: get tariffs
+            ############################################################################################################
+            print("Step 4: get tariffs")
+            tariff_dict = dict()
+            for energy_category_id in energy_category_list:
+                tariff_dict[energy_category_id] = tariff.get_energy_category_tariffs(shopfloor['cost_center_id'],
+                                                                                     energy_category_id,
+                                                                                     start_datetime_utc,
+                                                                                     end_datetime_utc)
+            ############################################################################################################
+            # Step 5: calculate billing by multiplying energy with tariff
+            ############################################################################################################
+            print("Step 5: calculate billing by multiplying energy with tariff")
+            billing_dict = dict()
+
+            if len(energy_dict) > 0:
+                for current_datetime_utc in energy_dict.keys():
+                    billing_dict[current_datetime_utc] = dict()
+                    for energy_category_id in energy_category_list:
+                        current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc)
+                        current_energy = energy_dict[current_datetime_utc].get(energy_category_id)
+                        if current_tariff is not None \
+                                and isinstance(current_tariff, Decimal) \
+                                and current_energy is not None \
+                                and isinstance(current_energy, Decimal):
+                            billing_dict[current_datetime_utc][energy_category_id] = \
+                                current_energy * current_tariff
+
+                    if len(billing_dict[current_datetime_utc]) == 0:
+                        del billing_dict[current_datetime_utc]
+
+            ############################################################################################################
+            # Step 6: save billing data to billing database
+            ############################################################################################################
+            print("Step 6: save billing data to billing database")
+
+            if len(billing_dict) > 0:
+                try:
+                    add_values = (" INSERT INTO tbl_shopfloor_input_category_hourly "
+                                  "             (shopfloor_id, "
+                                  "              energy_category_id, "
+                                  "              start_datetime_utc, "
+                                  "              actual_value) "
+                                  " VALUES  ")
+
+                    for current_datetime_utc in billing_dict:
+                        for energy_category_id in energy_category_list:
+                            current_billing = billing_dict[current_datetime_utc].get(energy_category_id)
+                            if current_billing is not None and isinstance(current_billing, Decimal):
+                                add_values += " (" + str(shopfloor['id']) + ","
+                                add_values += " " + str(energy_category_id) + ","
+                                add_values += "'" + current_datetime_utc.isoformat()[0:19] + "',"
+                                add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), "
+                    print("add_values:" + add_values)
+                    # trim ", " at the end of string and then execute
+                    cursor_billing_db.execute(add_values[:-2])
+                    cnx_billing_db.commit()
+                except Exception as e:
+                    logger.error("Error in step 6 of shopfloor_billing_input_category " + str(e))
+                    # break the for shopfloor loop
+                    break
+
+        # end of for shopfloor loop
+        if cnx_system_db:
+            cnx_system_db.close()
+        if cursor_system_db:
+            cursor_system_db.close()
+
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        if cursor_energy_db:
+            cursor_energy_db.close()
+
+        if cnx_billing_db:
+            cnx_billing_db.close()
+        if cursor_billing_db:
+            cursor_billing_db.close()
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of the outermost while loop

myems-aggregation/shopfloor_billing_input_item.py

@@ -0,0 +1,269 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+import tariff
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all shopfloors
+# for each shopfloor in list:
+#   Step 2: get the latest start_datetime_utc
+#   Step 3: get all energy input data since the latest start_datetime_utc
+#   Step 4: get tariffs
+#   Step 5: calculate billing by multiplying energy with tariff
+#   Step 6: save billing data to database
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all shopfloors
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of shopfloor_billing_input_item " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name, cost_center_id "
+                                     " FROM tbl_shopfloors "
+                                     " ORDER BY id ")
+            rows_shopfloors = cursor_system_db.fetchall()
+
+            if rows_shopfloors is None or len(rows_shopfloors) == 0:
+                print("Step 1.2: There isn't any shopfloors. ")
+                if cursor_system_db:
+                    cursor_system_db.close()
+                if cnx_system_db:
+                    cnx_system_db.close()
+                # sleep and continue the outermost while loop
+                time.sleep(60)
+                continue
+
+            shopfloor_list = list()
+            for row in rows_shopfloors:
+                shopfloor_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of shopfloor_billing_input_item " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Step 1.2: Got all shopfloors from MyEMS System Database")
+
+        cnx_energy_db = None
+        cursor_energy_db = None
+        try:
+            cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+            cursor_energy_db = cnx_energy_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.3 of shopfloor_billing_input_item " + str(e))
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Energy Database")
+
+        cnx_billing_db = None
+        cursor_billing_db = None
+        try:
+            cnx_billing_db = mysql.connector.connect(**config.myems_billing_db)
+            cursor_billing_db = cnx_billing_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.4 of shopfloor_billing_input_item " + str(e))
+            if cursor_billing_db:
+                cursor_billing_db.close()
+            if cnx_billing_db:
+                cnx_billing_db.close()
+
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Billing Database")
+
+        for shopfloor in shopfloor_list:
+
+            ############################################################################################################
+            # Step 2: get the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 2: get the latest start_datetime_utc from billing database for " + shopfloor['name'])
+            try:
+                cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) "
+                                          " FROM tbl_shopfloor_input_item_hourly "
+                                          " WHERE shopfloor_id = %s ",
+                                          (shopfloor['id'], ))
+                row_datetime = cursor_billing_db.fetchone()
+                start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+                start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+                if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+                    # replace second and microsecond with 0
+                    # note: do not replace minute in case of calculating in half hourly
+                    start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+                    # start from the next time slot
+                    start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+                print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19])
+            except Exception as e:
+                logger.error("Error in step 2 of shopfloor_billing_input_item " + str(e))
+                # break the for shopfloor loop
+                break
+
+            ############################################################################################################
+            # Step 3: get all energy input data since the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 3: get all energy input data since the latest start_datetime_utc")
+
+            query = (" SELECT start_datetime_utc, energy_item_id, actual_value "
+                     " FROM tbl_shopfloor_input_item_hourly "
+                     " WHERE shopfloor_id = %s AND start_datetime_utc >= %s "
+                     " ORDER BY id ")
+            cursor_energy_db.execute(query, (shopfloor['id'], start_datetime_utc, ))
+            rows_hourly = cursor_energy_db.fetchall()
+
+            if rows_hourly is None or len(rows_hourly) == 0:
+                print("Step 3: There isn't any energy input data to calculate. ")
+                # continue the for shopfloor loop
+                continue
+
+            energy_dict = dict()
+            energy_item_list = list()
+            end_datetime_utc = start_datetime_utc
+            for row_hourly in rows_hourly:
+                current_datetime_utc = row_hourly[0]
+                energy_item_id = row_hourly[1]
+
+                if energy_item_id not in energy_item_list:
+                    energy_item_list.append(energy_item_id)
+
+                actual_value = row_hourly[2]
+                if energy_dict.get(current_datetime_utc) is None:
+                    energy_dict[current_datetime_utc] = dict()
+                energy_dict[current_datetime_utc][energy_item_id] = actual_value
+                if current_datetime_utc > end_datetime_utc:
+                    end_datetime_utc = current_datetime_utc
+
+            ############################################################################################################
+            # Step 4: get tariffs
+            ############################################################################################################
+            print("Step 4: get tariffs")
+            tariff_dict = dict()
+            for energy_item_id in energy_item_list:
+                tariff_dict[energy_item_id] = tariff.get_energy_item_tariffs(shopfloor['cost_center_id'],
+                                                                             energy_item_id,
+                                                                             start_datetime_utc,
+                                                                             end_datetime_utc)
+            ############################################################################################################
+            # Step 5: calculate billing by multiplying energy with tariff
+            ############################################################################################################
+            print("Step 5: calculate billing by multiplying energy with tariff")
+            billing_dict = dict()
+
+            if len(energy_dict) > 0:
+                for current_datetime_utc in energy_dict.keys():
+                    billing_dict[current_datetime_utc] = dict()
+                    for energy_item_id in energy_item_list:
+                        current_tariff = tariff_dict[energy_item_id].get(current_datetime_utc)
+                        current_energy = energy_dict[current_datetime_utc].get(energy_item_id)
+                        if current_tariff is not None \
+                                and isinstance(current_tariff, Decimal) \
+                                and current_energy is not None \
+                                and isinstance(current_energy, Decimal):
+                            billing_dict[current_datetime_utc][energy_item_id] = \
+                                current_energy * current_tariff
+
+                    if len(billing_dict[current_datetime_utc]) == 0:
+                        del billing_dict[current_datetime_utc]
+
+            ############################################################################################################
+            # Step 6: save billing data to billing database
+            ############################################################################################################
+            print("Step 6: save billing data to billing database")
+
+            if len(billing_dict) > 0:
+                try:
+                    add_values = (" INSERT INTO tbl_shopfloor_input_item_hourly "
+                                  "             (shopfloor_id, "
+                                  "              energy_item_id, "
+                                  "              start_datetime_utc, "
+                                  "              actual_value) "
+                                  " VALUES  ")
+
+                    for current_datetime_utc in billing_dict:
+                        for energy_item_id in energy_item_list:
+                            current_billing = billing_dict[current_datetime_utc].get(energy_item_id)
+                            if current_billing is not None and isinstance(current_billing, Decimal):
+                                add_values += " (" + str(shopfloor['id']) + ","
+                                add_values += " " + str(energy_item_id) + ","
+                                add_values += "'" + current_datetime_utc.isoformat()[0:19] + "',"
+                                add_values += str(billing_dict[current_datetime_utc][energy_item_id]) + "), "
+                    print("add_values:" + add_values)
+                    # trim ", " at the end of string and then execute
+                    cursor_billing_db.execute(add_values[:-2])
+                    cnx_billing_db.commit()
+                except Exception as e:
+                    logger.error("Error in step 6 of shopfloor_billing_input_item " + str(e))
+                    # break the for shopfloor loop
+                    break
+
+        # end of for shopfloor loop
+        if cnx_system_db:
+            cnx_system_db.close()
+        if cursor_system_db:
+            cursor_system_db.close()
+
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        if cursor_energy_db:
+            cursor_energy_db.close()
+
+        if cnx_billing_db:
+            cnx_billing_db.close()
+        if cursor_billing_db:
+            cursor_billing_db.close()
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of the outermost while loop

myems-aggregation/shopfloor_energy_input_category.py

@@ -0,0 +1,615 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+from multiprocessing import Pool
+import random
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all shopfloors
+# Step 2: Create multiprocessing pool to call worker in parallel
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all shopfloors
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of shopfloor_energy_input_category.main " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name "
+                                     " FROM tbl_shopfloors "
+                                     " ORDER BY id ")
+            rows_shopfloors = cursor_system_db.fetchall()
+
+            if rows_shopfloors is None or len(rows_shopfloors) == 0:
+                print("There isn't any shopfloors ")
+                # sleep and continue the outer loop to reconnect the database
+                time.sleep(60)
+                continue
+
+            shopfloor_list = list()
+            for row in rows_shopfloors:
+                shopfloor_list.append({"id": row[0], "name": row[1]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of shopfloor_energy_input_category.main " + str(e))
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        finally:
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+
+        print("Got all shopfloors in MyEMS System Database")
+
+        # shuffle the shopfloor list for randomly calculating the meter hourly value
+        random.shuffle(shopfloor_list)
+
+        ################################################################################################################
+        # Step 2: Create multiprocessing pool to call worker in parallel
+        ################################################################################################################
+        p = Pool(processes=config.pool_size)
+        error_list = p.map(worker, shopfloor_list)
+        p.close()
+        p.join()
+
+        for error in error_list:
+            if error is not None and len(error) > 0:
+                logger.error(error)
+
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of outer while
+
+
+########################################################################################################################
+# PROCEDURES:
+#   Step 1: get all input meters associated with the shopfloor
+#   Step 2: get all input virtual meters associated with the shopfloor
+#   Step 3: get all input offline meters associated with the shopfloor
+#   Step 4: get all equipments associated with the shopfloor
+#   Step 5: determine start datetime and end datetime to aggregate
+#   Step 6: for each meter in list, get energy input data from energy database
+#   Step 7: for each virtual meter in list, get energy input data from energy database
+#   Step 8: for each offline meter in list, get energy input data from energy database
+#   Step 9: for each equipment in list, get energy input data from energy database
+#   Step 10: determine common time slot to aggregate
+#   Step 11: aggregate energy data in the common time slot by energy categories and hourly
+#   Step 12: save energy data to energy database
+#
+# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter
+########################################################################################################################
+
+def worker(shopfloor):
+    ####################################################################################################################
+    # Step 1: get all input meters associated with the shopfloor
+    ####################################################################################################################
+    print("Step 1: get all input meters associated with the shopfloor " + str(shopfloor['name']))
+
+    meter_list = list()
+    cnx_system_db = None
+    cursor_system_db = None
+    try:
+        cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+        cursor_system_db = cnx_system_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 1.1 of shopfloor_energy_input_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_meters m, tbl_shopfloors_meters tm "
+                                 " WHERE m.id = tm.meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND tm.shopfloor_id = %s ",
+                                 (shopfloor['id'],))
+        rows_meters = cursor_system_db.fetchall()
+
+        if rows_meters is not None and len(rows_meters) > 0:
+            for row in rows_meters:
+                meter_list.append({"id": row[0],
+                                   "name": row[1],
+                                   "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 1.2 of shopfloor_energy_input_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 2: get all input virtual meters associated with the shopfloor
+    ####################################################################################################################
+    print("Step 2: get all input virtual meters associated with the shopfloor")
+    virtual_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_virtual_meters m, tbl_shopfloors_virtual_meters tm "
+                                 " WHERE m.id = tm.virtual_meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND tm.shopfloor_id = %s ",
+                                 (shopfloor['id'],))
+        rows_virtual_meters = cursor_system_db.fetchall()
+
+        if rows_virtual_meters is not None and len(rows_virtual_meters) > 0:
+            for row in rows_virtual_meters:
+                virtual_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 2.1 of shopfloor_energy_input_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 3: get all input offline meters associated with the shopfloor
+    ####################################################################################################################
+    print("Step 3: get all input offline meters associated with the shopfloor")
+
+    offline_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_offline_meters m, tbl_shopfloors_offline_meters tm "
+                                 " WHERE m.id = tm.offline_meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND tm.shopfloor_id = %s ",
+                                 (shopfloor['id'],))
+        rows_offline_meters = cursor_system_db.fetchall()
+
+        if rows_offline_meters is not None and len(rows_offline_meters) > 0:
+            for row in rows_offline_meters:
+                offline_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 3.1 of shopfloor_energy_input_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 4: get all equipments associated with the shopfloor
+    ####################################################################################################################
+    print("Step 4: get all equipments associated with the shopfloor")
+
+    equipment_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT e.id, e.name "
+                                 " FROM tbl_equipments e, tbl_shopfloors_equipments se "
+                                 " WHERE e.id = se.equipment_id "
+                                 "       AND e.is_input_counted = true "
+                                 "       AND se.shopfloor_id = %s ",
+                                 (shopfloor['id'],))
+        rows_equipments = cursor_system_db.fetchall()
+
+        if rows_equipments is not None and len(rows_equipments) > 0:
+            for row in rows_equipments:
+                equipment_list.append({"id": row[0],
+                                       "name": row[1]})
+
+    except Exception as e:
+        error_string = "Error in step 4 of shopfloor_energy_input_category.worker " + str(e)
+        print(error_string)
+        return error_string
+    finally:
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+
+    ####################################################################################################################
+    # stop to the next shopfloor if this shopfloor is empty
+    ####################################################################################################################
+    if (meter_list is None or len(meter_list) == 0) and \
+            (virtual_meter_list is None or len(virtual_meter_list) == 0) and \
+            (offline_meter_list is None or len(offline_meter_list) == 0) and \
+            (equipment_list is None or len(equipment_list) == 0):
+        print("This is an empty shopfloor ")
+        return None
+
+    ####################################################################################################################
+    # Step 5: determine start datetime and end datetime to aggregate
+    ####################################################################################################################
+    print("Step 5: determine start datetime and end datetime to aggregate")
+    cnx_energy_db = None
+    cursor_energy_db = None
+    try:
+        cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+        cursor_energy_db = cnx_energy_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 5.1 of shopfloor_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        query = (" SELECT MAX(start_datetime_utc) "
+                 " FROM tbl_shopfloor_input_category_hourly "
+                 " WHERE shopfloor_id = %s ")
+        cursor_energy_db.execute(query, (shopfloor['id'],))
+        row_datetime = cursor_energy_db.fetchone()
+        start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+        start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+        if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+            # replace second and microsecond with 0
+            # note: do not replace minute in case of calculating in half hourly
+            start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+            # start from the next time slot
+            start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+        end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None)
+
+        print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]
+              + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19])
+
+    except Exception as e:
+        error_string = "Error in step 5.2 of shopfloor_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 6: for each meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_meter_hourly = dict()
+    try:
+        if meter_list is not None and len(meter_list) > 0:
+            for meter in meter_list:
+                meter_id = str(meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_meter_hourly "
+                         " WHERE meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_meter_hourly[meter_id] = None
+                else:
+                    energy_meter_hourly[meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1]
+    except Exception as e:
+        error_string = "Error in step 6.1 of shopfloor_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 7: for each virtual meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_virtual_meter_hourly = dict()
+    if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+        try:
+            for virtual_meter in virtual_meter_list:
+                virtual_meter_id = str(virtual_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_virtual_meter_hourly "
+                         " WHERE virtual_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_virtual_meter_hourly[virtual_meter_id] = None
+                else:
+                    energy_virtual_meter_hourly[virtual_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1]
+        except Exception as e:
+            error_string = "Error in step 7.1 of shopfloor_energy_input_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 8: for each offline meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_offline_meter_hourly = dict()
+    if offline_meter_list is not None and len(offline_meter_list) > 0:
+        try:
+            for offline_meter in offline_meter_list:
+                offline_meter_id = str(offline_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_offline_meter_hourly "
+                         " WHERE offline_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_offline_meter_hourly[offline_meter_id] = None
+                else:
+                    energy_offline_meter_hourly[offline_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1]
+
+        except Exception as e:
+            error_string = "Error in step 8.1 of shopfloor_energy_input_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 9: for each equipment in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_equipment_hourly = dict()
+    if equipment_list is not None and len(equipment_list) > 0:
+        try:
+            for equipment in equipment_list:
+                equipment_id = str(equipment['id'])
+                query = (" SELECT start_datetime_utc, energy_category_id, actual_value "
+                         " FROM tbl_equipment_input_category_hourly "
+                         " WHERE equipment_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (equipment_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_equipment_hourly[equipment_id] = None
+                else:
+                    energy_equipment_hourly[equipment_id] = dict()
+                    for row_value in rows_energy_values:
+                        current_datetime_utc = row_value[0]
+                        if current_datetime_utc not in energy_equipment_hourly[equipment_id]:
+                            energy_equipment_hourly[equipment_id][current_datetime_utc] = dict()
+                        energy_category_id = row_value[1]
+                        actual_value = row_value[2]
+                        energy_equipment_hourly[equipment_id][current_datetime_utc][energy_category_id] = \
+                            actual_value
+        except Exception as e:
+            error_string = "Error in step 9 of shopfloor_energy_input_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 10: determine common time slot to aggregate
+    ####################################################################################################################
+
+    common_start_datetime_utc = start_datetime_utc
+    common_end_datetime_utc = end_datetime_utc
+
+    print("Getting common time slot of energy values for all meters")
+    if energy_meter_hourly is not None and len(energy_meter_hourly) > 0:
+        for meter_id, energy_hourly in energy_meter_hourly.items():
+            if energy_hourly is None or len(energy_hourly) == 0:
+                common_start_datetime_utc = None
+                common_end_datetime_utc = None
+                break
+            else:
+                if common_start_datetime_utc < min(energy_hourly.keys()):
+                    common_start_datetime_utc = min(energy_hourly.keys())
+                if common_end_datetime_utc > max(energy_hourly.keys()):
+                    common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all virtual meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_virtual_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all offline meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_offline_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all equipments...")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_equipment_hourly is not None and len(energy_equipment_hourly) > 0:
+            for equipment_id, energy_hourly in energy_equipment_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \
+            (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \
+            (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0) and \
+            (energy_equipment_hourly is None or len(energy_equipment_hourly) == 0):
+        # There isn't any energy data
+        print("There isn't any energy data")
+        # continue the for shopfloor loop to the next shopfloor
+        print("continue the for shopfloor loop to the next shopfloor")
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        return None
+
+    print("common_start_datetime_utc: " + str(common_start_datetime_utc))
+    print("common_end_datetime_utc: " + str(common_end_datetime_utc))
+
+    ####################################################################################################################
+    # Step 11: aggregate energy data in the common time slot by energy categories and hourly
+    ####################################################################################################################
+
+    print("Step 11: aggregate energy data in the common time slot by energy categories and hourly")
+    aggregated_values = list()
+    try:
+        current_datetime_utc = common_start_datetime_utc
+        while common_start_datetime_utc is not None \
+                and common_end_datetime_utc is not None \
+                and current_datetime_utc <= common_end_datetime_utc:
+            aggregated_value = dict()
+            aggregated_value['start_datetime_utc'] = current_datetime_utc
+            aggregated_value['meta_data'] = dict()
+
+            if meter_list is not None and len(meter_list) > 0:
+                for meter in meter_list:
+                    meter_id = str(meter['id'])
+                    energy_category_id = meter['energy_category_id']
+                    actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+                for virtual_meter in virtual_meter_list:
+                    virtual_meter_id = str(virtual_meter['id'])
+                    energy_category_id = virtual_meter['energy_category_id']
+                    actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if offline_meter_list is not None and len(offline_meter_list) > 0:
+                for offline_meter in offline_meter_list:
+                    offline_meter_id = str(offline_meter['id'])
+                    energy_category_id = offline_meter['energy_category_id']
+                    actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if equipment_list is not None and len(equipment_list) > 0:
+                for equipment in equipment_list:
+                    equipment_id = str(equipment['id'])
+                    meta_data_dict = energy_equipment_hourly[equipment_id].get(current_datetime_utc, None)
+                    if meta_data_dict is not None and len(meta_data_dict) > 0:
+                        for energy_category_id, actual_value in meta_data_dict.items():
+                            aggregated_value['meta_data'][energy_category_id] = \
+                                aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            aggregated_values.append(aggregated_value)
+
+            current_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+    except Exception as e:
+        error_string = "Error in step 11 of shopfloor_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 12: save energy data to energy database
+    ####################################################################################################################
+    print("Step 12: save energy data to energy database")
+
+    if len(aggregated_values) > 0:
+        try:
+            add_values = (" INSERT INTO tbl_shopfloor_input_category_hourly "
+                          "             (shopfloor_id, "
+                          "              energy_category_id, "
+                          "              start_datetime_utc, "
+                          "              actual_value) "
+                          " VALUES  ")
+
+            for aggregated_value in aggregated_values:
+                for energy_category_id, actual_value in aggregated_value['meta_data'].items():
+                    add_values += " (" + str(shopfloor['id']) + ","
+                    add_values += " " + str(energy_category_id) + ","
+                    add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "',"
+                    add_values += str(actual_value) + "), "
+            print("add_values:" + add_values)
+            # trim ", " at the end of string and then execute
+            cursor_energy_db.execute(add_values[:-2])
+            cnx_energy_db.commit()
+
+        except Exception as e:
+            error_string = "Error in step 12.1 of shopfloor_energy_input_category.worker " + str(e)
+            print(error_string)
+            return error_string
+        finally:
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+    else:
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()

myems-aggregation/shopfloor_energy_input_item.py

@@ -0,0 +1,618 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+from multiprocessing import Pool
+import random
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all shopfloors
+# Step 2: Create multiprocessing pool to call worker in parallel
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all shopfloors
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of shopfloor_energy_input_item.main " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name "
+                                     " FROM tbl_shopfloors "
+                                     " ORDER BY id ")
+            rows_shopfloors = cursor_system_db.fetchall()
+
+            if rows_shopfloors is None or len(rows_shopfloors) == 0:
+                print("There isn't any shopfloors ")
+                # sleep and continue the outer loop to reconnect the database
+                time.sleep(60)
+                continue
+
+            shopfloor_list = list()
+            for row in rows_shopfloors:
+                shopfloor_list.append({"id": row[0], "name": row[1]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of shopfloor_energy_input_item.main " + str(e))
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        finally:
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+
+        print("Got all shopfloors in MyEMS System Database")
+
+        # shuffle the shopfloor list for randomly calculating the meter hourly value
+        random.shuffle(shopfloor_list)
+
+        ################################################################################################################
+        # Step 2: Create multiprocessing pool to call worker in parallel
+        ################################################################################################################
+        p = Pool(processes=config.pool_size)
+        error_list = p.map(worker, shopfloor_list)
+        p.close()
+        p.join()
+
+        for error in error_list:
+            if error is not None and len(error) > 0:
+                logger.error(error)
+
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of outer while
+
+
+########################################################################################################################
+# PROCEDURES:
+#   Step 1: get all input meters associated with the shopfloor
+#   Step 2: get all input virtual meters associated with the shopfloor
+#   Step 3: get all input offline meters associated with the shopfloor
+#   Step 4: get all equipments associated with the shopfloor
+#   Step 5: determine start datetime and end datetime to aggregate
+#   Step 6: for each meter in list, get energy input data from energy database
+#   Step 7: for each virtual meter in list, get energy input data from energy database
+#   Step 8: for each offline meter in list, get energy input data from energy database
+#   Step 9: for each equipment in list, get energy input data from energy database
+#   Step 10: determine common time slot to aggregate
+#   Step 11: aggregate energy data in the common time slot by energy items and hourly
+#   Step 12: save energy data to energy database
+#
+# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter
+########################################################################################################################
+
+def worker(shopfloor):
+    ####################################################################################################################
+    # Step 1: get all input meters associated with the shopfloor
+    ####################################################################################################################
+    print("Step 1: get all input meters associated with the shopfloor " + str(shopfloor['name']))
+
+    meter_list = list()
+    cnx_system_db = None
+    cursor_system_db = None
+    try:
+        cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+        cursor_system_db = cnx_system_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 1.1 of shopfloor_energy_input_item.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id "
+                                 " FROM tbl_meters m, tbl_shopfloors_meters tm "
+                                 " WHERE m.id = tm.meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND m.energy_item_id is NOT NULL "
+                                 "       AND tm.shopfloor_id = %s ",
+                                 (shopfloor['id'],))
+        rows_meters = cursor_system_db.fetchall()
+
+        if rows_meters is not None and len(rows_meters) > 0:
+            for row in rows_meters:
+                meter_list.append({"id": row[0],
+                                   "name": row[1],
+                                   "energy_item_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 1.2 of shopfloor_energy_input_item.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 2: get all input virtual meters associated with the shopfloor
+    ####################################################################################################################
+    print("Step 2: get all input virtual meters associated with the shopfloor")
+    virtual_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id "
+                                 " FROM tbl_virtual_meters m, tbl_shopfloors_virtual_meters tm "
+                                 " WHERE m.id = tm.virtual_meter_id "
+                                 "       AND m.energy_item_id is NOT NULL "
+                                 "       AND m.is_counted = true "
+                                 "       AND tm.shopfloor_id = %s ",
+                                 (shopfloor['id'],))
+        rows_virtual_meters = cursor_system_db.fetchall()
+
+        if rows_virtual_meters is not None and len(rows_virtual_meters) > 0:
+            for row in rows_virtual_meters:
+                virtual_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_item_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 2.1 of shopfloor_energy_input_item.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 3: get all input offline meters associated with the shopfloor
+    ####################################################################################################################
+    print("Step 3: get all input offline meters associated with the shopfloor")
+
+    offline_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id "
+                                 " FROM tbl_offline_meters m, tbl_shopfloors_offline_meters tm "
+                                 " WHERE m.id = tm.offline_meter_id "
+                                 "       AND m.energy_item_id is NOT NULL "
+                                 "       AND m.is_counted = true "
+                                 "       AND tm.shopfloor_id = %s ",
+                                 (shopfloor['id'],))
+        rows_offline_meters = cursor_system_db.fetchall()
+
+        if rows_offline_meters is not None and len(rows_offline_meters) > 0:
+            for row in rows_offline_meters:
+                offline_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_item_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 3.1 of shopfloor_energy_input_item.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 4: get all equipments associated with the shopfloor
+    ####################################################################################################################
+    print("Step 4: get all equipments associated with the shopfloor")
+
+    equipment_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT e.id, e.name "
+                                 " FROM tbl_equipments e, tbl_shopfloors_equipments se "
+                                 " WHERE e.id = se.equipment_id "
+                                 "       AND e.is_input_counted = true "
+                                 "       AND se.shopfloor_id = %s ",
+                                 (shopfloor['id'],))
+        rows_equipments = cursor_system_db.fetchall()
+
+        if rows_equipments is not None and len(rows_equipments) > 0:
+            for row in rows_equipments:
+                equipment_list.append({"id": row[0],
+                                       "name": row[1]})
+
+    except Exception as e:
+        error_string = "Error in step 4 of shopfloor_energy_input_item.worker " + str(e)
+        print(error_string)
+        return error_string
+    finally:
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+
+    ####################################################################################################################
+    # stop to the next shopfloor if this shopfloor is empty
+    ####################################################################################################################
+    if (meter_list is None or len(meter_list) == 0) and \
+            (virtual_meter_list is None or len(virtual_meter_list) == 0) and \
+            (offline_meter_list is None or len(offline_meter_list) == 0) and \
+            (equipment_list is None or len(equipment_list) == 0):
+        print("This is an empty shopfloor ")
+        return None
+
+    ####################################################################################################################
+    # Step 5: determine start datetime and end datetime to aggregate
+    ####################################################################################################################
+    print("Step 5: determine start datetime and end datetime to aggregate")
+    cnx_energy_db = None
+    cursor_energy_db = None
+    try:
+        cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+        cursor_energy_db = cnx_energy_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 5.1 of shopfloor_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        query = (" SELECT MAX(start_datetime_utc) "
+                 " FROM tbl_shopfloor_input_item_hourly "
+                 " WHERE shopfloor_id = %s ")
+        cursor_energy_db.execute(query, (shopfloor['id'],))
+        row_datetime = cursor_energy_db.fetchone()
+        start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+        start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+        if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+            # replace second and microsecond with 0
+            # note: do not replace minute in case of calculating in half hourly
+            start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+            # start from the next time slot
+            start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+        end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None)
+
+        print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]
+              + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19])
+
+    except Exception as e:
+        error_string = "Error in step 5.2 of shopfloor_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 6: for each meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_meter_hourly = dict()
+    try:
+        if meter_list is not None and len(meter_list) > 0:
+            for meter in meter_list:
+                meter_id = str(meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_meter_hourly "
+                         " WHERE meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_meter_hourly[meter_id] = None
+                else:
+                    energy_meter_hourly[meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1]
+    except Exception as e:
+        error_string = "Error in step 6.1 of shopfloor_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 7: for each virtual meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_virtual_meter_hourly = dict()
+    if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+        try:
+            for virtual_meter in virtual_meter_list:
+                virtual_meter_id = str(virtual_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_virtual_meter_hourly "
+                         " WHERE virtual_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_virtual_meter_hourly[virtual_meter_id] = None
+                else:
+                    energy_virtual_meter_hourly[virtual_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1]
+        except Exception as e:
+            error_string = "Error in step 7.1 of shopfloor_energy_input_item.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 8: for each offline meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_offline_meter_hourly = dict()
+    if offline_meter_list is not None and len(offline_meter_list) > 0:
+        try:
+            for offline_meter in offline_meter_list:
+                offline_meter_id = str(offline_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_offline_meter_hourly "
+                         " WHERE offline_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_offline_meter_hourly[offline_meter_id] = None
+                else:
+                    energy_offline_meter_hourly[offline_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1]
+
+        except Exception as e:
+            error_string = "Error in step 8.1 of shopfloor_energy_input_item.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 9: for each equipment in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_equipment_hourly = dict()
+    if equipment_list is not None and len(equipment_list) > 0:
+        try:
+            for equipment in equipment_list:
+                equipment_id = str(equipment['id'])
+                query = (" SELECT start_datetime_utc, energy_item_id, actual_value "
+                         " FROM tbl_equipment_input_item_hourly "
+                         " WHERE equipment_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (equipment_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_equipment_hourly[equipment_id] = None
+                else:
+                    energy_equipment_hourly[equipment_id] = dict()
+                    for row_value in rows_energy_values:
+                        current_datetime_utc = row_value[0]
+                        if current_datetime_utc not in energy_equipment_hourly[equipment_id]:
+                            energy_equipment_hourly[equipment_id][current_datetime_utc] = dict()
+                        energy_item_id = row_value[1]
+                        actual_value = row_value[2]
+                        energy_equipment_hourly[equipment_id][current_datetime_utc][energy_item_id] = \
+                            actual_value
+        except Exception as e:
+            error_string = "Error in step 9 of shopfloor_energy_input_item.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 10: determine common time slot to aggregate
+    ####################################################################################################################
+
+    common_start_datetime_utc = start_datetime_utc
+    common_end_datetime_utc = end_datetime_utc
+
+    print("Getting common time slot of energy values for all meters")
+    if energy_meter_hourly is not None and len(energy_meter_hourly) > 0:
+        for meter_id, energy_hourly in energy_meter_hourly.items():
+            if energy_hourly is None or len(energy_hourly) == 0:
+                common_start_datetime_utc = None
+                common_end_datetime_utc = None
+                break
+            else:
+                if common_start_datetime_utc < min(energy_hourly.keys()):
+                    common_start_datetime_utc = min(energy_hourly.keys())
+                if common_end_datetime_utc > max(energy_hourly.keys()):
+                    common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all virtual meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_virtual_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all offline meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_offline_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all equipments...")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_equipment_hourly is not None and len(energy_equipment_hourly) > 0:
+            for equipment_id, energy_hourly in energy_equipment_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \
+            (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \
+            (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0) and \
+            (energy_equipment_hourly is None or len(energy_equipment_hourly) == 0):
+        # There isn't any energy data
+        print("There isn't any energy data")
+        # continue the for shopfloor loop to the next shopfloor
+        print("continue the for shopfloor loop to the next shopfloor")
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        return None
+
+    print("common_start_datetime_utc: " + str(common_start_datetime_utc))
+    print("common_end_datetime_utc: " + str(common_end_datetime_utc))
+
+    ####################################################################################################################
+    # Step 11: aggregate energy data in the common time slot by energy items and hourly
+    ####################################################################################################################
+
+    print("Step 11: aggregate energy data in the common time slot by energy items and hourly")
+    aggregated_values = list()
+    try:
+        current_datetime_utc = common_start_datetime_utc
+        while common_start_datetime_utc is not None \
+                and common_end_datetime_utc is not None \
+                and current_datetime_utc <= common_end_datetime_utc:
+            aggregated_value = dict()
+            aggregated_value['start_datetime_utc'] = current_datetime_utc
+            aggregated_value['meta_data'] = dict()
+
+            if meter_list is not None and len(meter_list) > 0:
+                for meter in meter_list:
+                    meter_id = str(meter['id'])
+                    energy_item_id = meter['energy_item_id']
+                    actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_item_id] = \
+                        aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value
+
+            if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+                for virtual_meter in virtual_meter_list:
+                    virtual_meter_id = str(virtual_meter['id'])
+                    energy_item_id = virtual_meter['energy_item_id']
+                    actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_item_id] = \
+                        aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value
+
+            if offline_meter_list is not None and len(offline_meter_list) > 0:
+                for offline_meter in offline_meter_list:
+                    offline_meter_id = str(offline_meter['id'])
+                    energy_item_id = offline_meter['energy_item_id']
+                    actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_item_id] = \
+                        aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value
+
+            if equipment_list is not None and len(equipment_list) > 0:
+                for equipment in equipment_list:
+                    equipment_id = str(equipment['id'])
+                    meta_data_dict = energy_equipment_hourly[equipment_id].get(current_datetime_utc, None)
+                    if meta_data_dict is not None and len(meta_data_dict) > 0:
+                        for energy_item_id, actual_value in meta_data_dict.items():
+                            aggregated_value['meta_data'][energy_item_id] = \
+                                aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value
+
+            aggregated_values.append(aggregated_value)
+
+            current_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+    except Exception as e:
+        error_string = "Error in step 11 of shopfloor_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 12: save energy data to energy database
+    ####################################################################################################################
+    print("Step 12: save energy data to energy database")
+
+    if len(aggregated_values) > 0:
+        try:
+            add_values = (" INSERT INTO tbl_shopfloor_input_item_hourly "
+                          "             (shopfloor_id, "
+                          "              energy_item_id, "
+                          "              start_datetime_utc, "
+                          "              actual_value) "
+                          " VALUES  ")
+
+            for aggregated_value in aggregated_values:
+                for energy_item_id, actual_value in aggregated_value['meta_data'].items():
+                    add_values += " (" + str(shopfloor['id']) + ","
+                    add_values += " " + str(energy_item_id) + ","
+                    add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "',"
+                    add_values += str(actual_value) + "), "
+            print("add_values:" + add_values)
+            # trim ", " at the end of string and then execute
+            cursor_energy_db.execute(add_values[:-2])
+            cnx_energy_db.commit()
+
+        except Exception as e:
+            error_string = "Error in step 12.1 of shopfloor_energy_input_item.worker " + str(e)
+            print(error_string)
+            return error_string
+        finally:
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+    else:
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()

myems-aggregation/space_billing_input_category.py

@@ -0,0 +1,269 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+import tariff
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all spaces
+# for each space in list:
+#   Step 2: get the latest start_datetime_utc
+#   Step 3: get all energy input data since the latest start_datetime_utc
+#   Step 4: get tariffs
+#   Step 5: calculate billing by multiplying energy with tariff
+#   Step 6: save billing data to database
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all spaces
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of space_billing_input_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name, cost_center_id "
+                                     " FROM tbl_spaces "
+                                     " ORDER BY id ")
+            rows_spaces = cursor_system_db.fetchall()
+
+            if rows_spaces is None or len(rows_spaces) == 0:
+                print("Step 1.2: There isn't any spaces. ")
+                if cursor_system_db:
+                    cursor_system_db.close()
+                if cnx_system_db:
+                    cnx_system_db.close()
+                # sleep and continue the outermost while loop
+                time.sleep(60)
+                continue
+
+            space_list = list()
+            for row in rows_spaces:
+                space_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of space_billing_input_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Step 1.2: Got all spaces from MyEMS System Database")
+
+        cnx_energy_db = None
+        cursor_energy_db = None
+        try:
+            cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+            cursor_energy_db = cnx_energy_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.3 of space_billing_input_category " + str(e))
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Energy Database")
+
+        cnx_billing_db = None
+        cursor_billing_db = None
+        try:
+            cnx_billing_db = mysql.connector.connect(**config.myems_billing_db)
+            cursor_billing_db = cnx_billing_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.4 of space_billing_input_category " + str(e))
+            if cursor_billing_db:
+                cursor_billing_db.close()
+            if cnx_billing_db:
+                cnx_billing_db.close()
+
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Billing Database")
+
+        for space in space_list:
+
+            ############################################################################################################
+            # Step 2: get the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 2: get the latest start_datetime_utc from billing database for " + space['name'])
+            try:
+                cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) "
+                                          " FROM tbl_space_input_category_hourly "
+                                          " WHERE space_id = %s ",
+                                          (space['id'], ))
+                row_datetime = cursor_billing_db.fetchone()
+                start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+                start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+                if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+                    # replace second and microsecond with 0
+                    # note: do not replace minute in case of calculating in half hourly
+                    start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+                    # start from the next time slot
+                    start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+                print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19])
+            except Exception as e:
+                logger.error("Error in step 2 of space_billing_input_category " + str(e))
+                # break the for space loop
+                break
+
+            ############################################################################################################
+            # Step 3: get all energy input data since the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 3: get all energy input data since the latest start_datetime_utc")
+
+            query = (" SELECT start_datetime_utc, energy_category_id, actual_value "
+                     " FROM tbl_space_input_category_hourly "
+                     " WHERE space_id = %s AND start_datetime_utc >= %s "
+                     " ORDER BY id ")
+            cursor_energy_db.execute(query, (space['id'], start_datetime_utc, ))
+            rows_hourly = cursor_energy_db.fetchall()
+
+            if rows_hourly is None or len(rows_hourly) == 0:
+                print("Step 3: There isn't any energy input data to calculate. ")
+                # continue the for space loop
+                continue
+
+            energy_dict = dict()
+            energy_category_list = list()
+            end_datetime_utc = start_datetime_utc
+            for row_hourly in rows_hourly:
+                current_datetime_utc = row_hourly[0]
+                energy_category_id = row_hourly[1]
+
+                if energy_category_id not in energy_category_list:
+                    energy_category_list.append(energy_category_id)
+
+                actual_value = row_hourly[2]
+                if energy_dict.get(current_datetime_utc) is None:
+                    energy_dict[current_datetime_utc] = dict()
+                energy_dict[current_datetime_utc][energy_category_id] = actual_value
+                if current_datetime_utc > end_datetime_utc:
+                    end_datetime_utc = current_datetime_utc
+
+            ############################################################################################################
+            # Step 4: get tariffs
+            ############################################################################################################
+            print("Step 4: get tariffs")
+            tariff_dict = dict()
+            for energy_category_id in energy_category_list:
+                tariff_dict[energy_category_id] = tariff.get_energy_category_tariffs(space['cost_center_id'],
+                                                                                     energy_category_id,
+                                                                                     start_datetime_utc,
+                                                                                     end_datetime_utc)
+            ############################################################################################################
+            # Step 5: calculate billing by multiplying energy with tariff
+            ############################################################################################################
+            print("Step 5: calculate billing by multiplying energy with tariff")
+            billing_dict = dict()
+
+            if len(energy_dict) > 0:
+                for current_datetime_utc in energy_dict.keys():
+                    billing_dict[current_datetime_utc] = dict()
+                    for energy_category_id in energy_category_list:
+                        current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc)
+                        current_energy = energy_dict[current_datetime_utc].get(energy_category_id)
+                        if current_tariff is not None \
+                                and isinstance(current_tariff, Decimal) \
+                                and current_energy is not None \
+                                and isinstance(current_energy, Decimal):
+                            billing_dict[current_datetime_utc][energy_category_id] = \
+                                current_energy * current_tariff
+
+                    if len(billing_dict[current_datetime_utc]) == 0:
+                        del billing_dict[current_datetime_utc]
+
+            ############################################################################################################
+            # Step 6: save billing data to billing database
+            ############################################################################################################
+            print("Step 6: save billing data to billing database")
+
+            if len(billing_dict) > 0:
+                try:
+                    add_values = (" INSERT INTO tbl_space_input_category_hourly "
+                                  "             (space_id, "
+                                  "              energy_category_id, "
+                                  "              start_datetime_utc, "
+                                  "              actual_value) "
+                                  " VALUES  ")
+
+                    for current_datetime_utc in billing_dict:
+                        for energy_category_id in energy_category_list:
+                            current_billing = billing_dict[current_datetime_utc].get(energy_category_id)
+                            if current_billing is not None and isinstance(current_billing, Decimal):
+                                add_values += " (" + str(space['id']) + ","
+                                add_values += " " + str(energy_category_id) + ","
+                                add_values += "'" + current_datetime_utc.isoformat()[0:19] + "',"
+                                add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), "
+                    print("add_values:" + add_values)
+                    # trim ", " at the end of string and then execute
+                    cursor_billing_db.execute(add_values[:-2])
+                    cnx_billing_db.commit()
+                except Exception as e:
+                    logger.error("Error in step 6 of space_billing_input_category " + str(e))
+                    # break the for space loop
+                    break
+
+        # end of for space loop
+        if cnx_system_db:
+            cnx_system_db.close()
+        if cursor_system_db:
+            cursor_system_db.close()
+
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        if cursor_energy_db:
+            cursor_energy_db.close()
+
+        if cnx_billing_db:
+            cnx_billing_db.close()
+        if cursor_billing_db:
+            cursor_billing_db.close()
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of the outermost while loop

myems-aggregation/space_billing_input_item.py

@@ -0,0 +1,269 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+import tariff
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all spaces
+# for each space in list:
+#   Step 2: get the latest start_datetime_utc
+#   Step 3: get all energy input data since the latest start_datetime_utc
+#   Step 4: get tariffs
+#   Step 5: calculate billing by multiplying energy with tariff
+#   Step 6: save billing data to database
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all spaces
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of space_billing_input_item " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name, cost_center_id "
+                                     " FROM tbl_spaces "
+                                     " ORDER BY id ")
+            rows_spaces = cursor_system_db.fetchall()
+
+            if rows_spaces is None or len(rows_spaces) == 0:
+                print("Step 1.2: There isn't any spaces. ")
+                if cursor_system_db:
+                    cursor_system_db.close()
+                if cnx_system_db:
+                    cnx_system_db.close()
+                # sleep and continue the outermost while loop
+                time.sleep(60)
+                continue
+
+            space_list = list()
+            for row in rows_spaces:
+                space_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of space_billing_input_item " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Step 1.2: Got all spaces from MyEMS System Database")
+
+        cnx_energy_db = None
+        cursor_energy_db = None
+        try:
+            cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+            cursor_energy_db = cnx_energy_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.3 of space_billing_input_item " + str(e))
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Energy Database")
+
+        cnx_billing_db = None
+        cursor_billing_db = None
+        try:
+            cnx_billing_db = mysql.connector.connect(**config.myems_billing_db)
+            cursor_billing_db = cnx_billing_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.4 of space_billing_input_item " + str(e))
+            if cursor_billing_db:
+                cursor_billing_db.close()
+            if cnx_billing_db:
+                cnx_billing_db.close()
+
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Billing Database")
+
+        for space in space_list:
+
+            ############################################################################################################
+            # Step 2: get the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 2: get the latest start_datetime_utc from billing database for " + space['name'])
+            try:
+                cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) "
+                                          " FROM tbl_space_input_item_hourly "
+                                          " WHERE space_id = %s ",
+                                          (space['id'], ))
+                row_datetime = cursor_billing_db.fetchone()
+                start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+                start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+                if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+                    # replace second and microsecond with 0
+                    # note: do not replace minute in case of calculating in half hourly
+                    start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+                    # start from the next time slot
+                    start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+                print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19])
+            except Exception as e:
+                logger.error("Error in step 2 of space_billing_input_item " + str(e))
+                # break the for space loop
+                break
+
+            ############################################################################################################
+            # Step 3: get all energy input data since the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 3: get all energy input data since the latest start_datetime_utc")
+
+            query = (" SELECT start_datetime_utc, energy_item_id, actual_value "
+                     " FROM tbl_space_input_item_hourly "
+                     " WHERE space_id = %s AND start_datetime_utc >= %s "
+                     " ORDER BY id ")
+            cursor_energy_db.execute(query, (space['id'], start_datetime_utc, ))
+            rows_hourly = cursor_energy_db.fetchall()
+
+            if rows_hourly is None or len(rows_hourly) == 0:
+                print("Step 3: There isn't any energy input data to calculate. ")
+                # continue the for space loop
+                continue
+
+            energy_dict = dict()
+            energy_item_list = list()
+            end_datetime_utc = start_datetime_utc
+            for row_hourly in rows_hourly:
+                current_datetime_utc = row_hourly[0]
+                energy_item_id = row_hourly[1]
+
+                if energy_item_id not in energy_item_list:
+                    energy_item_list.append(energy_item_id)
+
+                actual_value = row_hourly[2]
+                if energy_dict.get(current_datetime_utc) is None:
+                    energy_dict[current_datetime_utc] = dict()
+                energy_dict[current_datetime_utc][energy_item_id] = actual_value
+                if current_datetime_utc > end_datetime_utc:
+                    end_datetime_utc = current_datetime_utc
+
+            ############################################################################################################
+            # Step 4: get tariffs
+            ############################################################################################################
+            print("Step 4: get tariffs")
+            tariff_dict = dict()
+            for energy_item_id in energy_item_list:
+                tariff_dict[energy_item_id] = tariff.get_energy_item_tariffs(space['cost_center_id'],
+                                                                             energy_item_id,
+                                                                             start_datetime_utc,
+                                                                             end_datetime_utc)
+            ############################################################################################################
+            # Step 5: calculate billing by multiplying energy with tariff
+            ############################################################################################################
+            print("Step 5: calculate billing by multiplying energy with tariff")
+            billing_dict = dict()
+
+            if len(energy_dict) > 0:
+                for current_datetime_utc in energy_dict.keys():
+                    billing_dict[current_datetime_utc] = dict()
+                    for energy_item_id in energy_item_list:
+                        current_tariff = tariff_dict[energy_item_id].get(current_datetime_utc)
+                        current_energy = energy_dict[current_datetime_utc].get(energy_item_id)
+                        if current_tariff is not None \
+                                and isinstance(current_tariff, Decimal) \
+                                and current_energy is not None \
+                                and isinstance(current_energy, Decimal):
+                            billing_dict[current_datetime_utc][energy_item_id] = \
+                                current_energy * current_tariff
+
+                    if len(billing_dict[current_datetime_utc]) == 0:
+                        del billing_dict[current_datetime_utc]
+
+            ############################################################################################################
+            # Step 6: save billing data to billing database
+            ############################################################################################################
+            print("Step 6: save billing data to billing database")
+
+            if len(billing_dict) > 0:
+                try:
+                    add_values = (" INSERT INTO tbl_space_input_item_hourly "
+                                  "             (space_id, "
+                                  "              energy_item_id, "
+                                  "              start_datetime_utc, "
+                                  "              actual_value) "
+                                  " VALUES  ")
+
+                    for current_datetime_utc in billing_dict:
+                        for energy_item_id in energy_item_list:
+                            current_billing = billing_dict[current_datetime_utc].get(energy_item_id)
+                            if current_billing is not None and isinstance(current_billing, Decimal):
+                                add_values += " (" + str(space['id']) + ","
+                                add_values += " " + str(energy_item_id) + ","
+                                add_values += "'" + current_datetime_utc.isoformat()[0:19] + "',"
+                                add_values += str(billing_dict[current_datetime_utc][energy_item_id]) + "), "
+                    print("add_values:" + add_values)
+                    # trim ", " at the end of string and then execute
+                    cursor_billing_db.execute(add_values[:-2])
+                    cnx_billing_db.commit()
+                except Exception as e:
+                    logger.error("Error in step 6 of space_billing_input_item " + str(e))
+                    # break the for space loop
+                    break
+
+        # end of for space loop
+        if cnx_system_db:
+            cnx_system_db.close()
+        if cursor_system_db:
+            cursor_system_db.close()
+
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        if cursor_energy_db:
+            cursor_energy_db.close()
+
+        if cnx_billing_db:
+            cnx_billing_db.close()
+        if cursor_billing_db:
+            cursor_billing_db.close()
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of the outermost while loop

myems-aggregation/space_billing_output_category.py

@@ -0,0 +1,269 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+import tariff
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all spaces
+# for each space in list:
+#   Step 2: get the latest start_datetime_utc
+#   Step 3: get all energy input data since the latest start_datetime_utc
+#   Step 4: get tariffs
+#   Step 5: calculate billing by multiplying energy with tariff
+#   Step 6: save billing data to database
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all spaces
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of space_billing_output_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name, cost_center_id "
+                                     " FROM tbl_spaces "
+                                     " ORDER BY id ")
+            rows_spaces = cursor_system_db.fetchall()
+
+            if rows_spaces is None or len(rows_spaces) == 0:
+                print("Step 1.2: There isn't any spaces. ")
+                if cursor_system_db:
+                    cursor_system_db.close()
+                if cnx_system_db:
+                    cnx_system_db.close()
+                # sleep and continue the outermost while loop
+                time.sleep(60)
+                continue
+
+            space_list = list()
+            for row in rows_spaces:
+                space_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of space_billing_output_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Step 1.2: Got all spaces from MyEMS System Database")
+
+        cnx_energy_db = None
+        cursor_energy_db = None
+        try:
+            cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+            cursor_energy_db = cnx_energy_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.3 of space_billing_output_category " + str(e))
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Energy Database")
+
+        cnx_billing_db = None
+        cursor_billing_db = None
+        try:
+            cnx_billing_db = mysql.connector.connect(**config.myems_billing_db)
+            cursor_billing_db = cnx_billing_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.4 of space_billing_output_category " + str(e))
+            if cursor_billing_db:
+                cursor_billing_db.close()
+            if cnx_billing_db:
+                cnx_billing_db.close()
+
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Billing Database")
+
+        for space in space_list:
+
+            ############################################################################################################
+            # Step 2: get the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 2: get the latest start_datetime_utc from billing database for " + space['name'])
+            try:
+                cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) "
+                                          " FROM tbl_space_output_category_hourly "
+                                          " WHERE space_id = %s ",
+                                          (space['id'], ))
+                row_datetime = cursor_billing_db.fetchone()
+                start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+                start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+                if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+                    # replace second and microsecond with 0
+                    # note: do not replace minute in case of calculating in half hourly
+                    start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+                    # start from the next time slot
+                    start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+                print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19])
+            except Exception as e:
+                logger.error("Error in step 2 of space_billing_output_category " + str(e))
+                # break the for space loop
+                break
+
+            ############################################################################################################
+            # Step 3: get all energy output data since the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 3: get all energy output data since the latest start_datetime_utc")
+
+            query = (" SELECT start_datetime_utc, energy_category_id, actual_value "
+                     " FROM tbl_space_output_category_hourly "
+                     " WHERE space_id = %s AND start_datetime_utc >= %s "
+                     " ORDER BY id ")
+            cursor_energy_db.execute(query, (space['id'], start_datetime_utc, ))
+            rows_hourly = cursor_energy_db.fetchall()
+
+            if rows_hourly is None or len(rows_hourly) == 0:
+                print("Step 3: There isn't any energy output data to calculate. ")
+                # continue the for space loop
+                continue
+
+            energy_dict = dict()
+            energy_category_list = list()
+            end_datetime_utc = start_datetime_utc
+            for row_hourly in rows_hourly:
+                current_datetime_utc = row_hourly[0]
+                energy_category_id = row_hourly[1]
+
+                if energy_category_id not in energy_category_list:
+                    energy_category_list.append(energy_category_id)
+
+                actual_value = row_hourly[2]
+                if energy_dict.get(current_datetime_utc) is None:
+                    energy_dict[current_datetime_utc] = dict()
+                energy_dict[current_datetime_utc][energy_category_id] = actual_value
+                if current_datetime_utc > end_datetime_utc:
+                    end_datetime_utc = current_datetime_utc
+
+            ############################################################################################################
+            # Step 4: get tariffs
+            ############################################################################################################
+            print("Step 4: get tariffs")
+            tariff_dict = dict()
+            for energy_category_id in energy_category_list:
+                tariff_dict[energy_category_id] = tariff.get_energy_category_tariffs(space['cost_center_id'],
+                                                                                     energy_category_id,
+                                                                                     start_datetime_utc,
+                                                                                     end_datetime_utc)
+            ############################################################################################################
+            # Step 5: calculate billing by multiplying energy with tariff
+            ############################################################################################################
+            print("Step 5: calculate billing by multiplying energy with tariff")
+            billing_dict = dict()
+
+            if len(energy_dict) > 0:
+                for current_datetime_utc in energy_dict.keys():
+                    billing_dict[current_datetime_utc] = dict()
+                    for energy_category_id in energy_category_list:
+                        current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc)
+                        current_energy = energy_dict[current_datetime_utc].get(energy_category_id)
+                        if current_tariff is not None \
+                                and isinstance(current_tariff, Decimal) \
+                                and current_energy is not None \
+                                and isinstance(current_energy, Decimal):
+                            billing_dict[current_datetime_utc][energy_category_id] = \
+                                current_energy * current_tariff
+
+                    if len(billing_dict[current_datetime_utc]) == 0:
+                        del billing_dict[current_datetime_utc]
+
+            ############################################################################################################
+            # Step 6: save billing data to billing database
+            ############################################################################################################
+            print("Step 6: save billing data to billing database")
+
+            if len(billing_dict) > 0:
+                try:
+                    add_values = (" INSERT INTO tbl_space_output_category_hourly "
+                                  "             (space_id, "
+                                  "              energy_category_id, "
+                                  "              start_datetime_utc, "
+                                  "              actual_value) "
+                                  " VALUES  ")
+
+                    for current_datetime_utc in billing_dict:
+                        for energy_category_id in energy_category_list:
+                            current_billing = billing_dict[current_datetime_utc].get(energy_category_id)
+                            if current_billing is not None and isinstance(current_billing, Decimal):
+                                add_values += " (" + str(space['id']) + ","
+                                add_values += " " + str(energy_category_id) + ","
+                                add_values += "'" + current_datetime_utc.isoformat()[0:19] + "',"
+                                add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), "
+                    print("add_values:" + add_values)
+                    # trim ", " at the end of string and then execute
+                    cursor_billing_db.execute(add_values[:-2])
+                    cnx_billing_db.commit()
+                except Exception as e:
+                    logger.error("Error in step 6 of space_billing_output_category " + str(e))
+                    # break the for space loop
+                    break
+
+        # end of for space loop
+        if cnx_system_db:
+            cnx_system_db.close()
+        if cursor_system_db:
+            cursor_system_db.close()
+
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        if cursor_energy_db:
+            cursor_energy_db.close()
+
+        if cnx_billing_db:
+            cnx_billing_db.close()
+        if cursor_billing_db:
+            cursor_billing_db.close()
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of the outermost while loop

myems-aggregation/space_energy_output_category.py

@@ -0,0 +1,533 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+from multiprocessing import Pool
+import random
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all spaces
+# Step 2: Create multiprocessing pool to call worker in parallel
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all spaces
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of space_energy_output_category.main " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name "
+                                     " FROM tbl_spaces "
+                                     " ORDER BY id ")
+            rows_spaces = cursor_system_db.fetchall()
+
+            if rows_spaces is None or len(rows_spaces) == 0:
+                print("There isn't any spaces ")
+                # sleep and continue the outer loop to reconnect the database
+                time.sleep(60)
+                continue
+
+            space_list = list()
+            for row in rows_spaces:
+                space_list.append({"id": row[0], "name": row[1]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of space_energy_output_category.main " + str(e))
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        finally:
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+
+        print("Got all spaces in MyEMS System Database")
+
+        # shuffle the space list for randomly calculating the meter hourly value
+        random.shuffle(space_list)
+
+        ################################################################################################################
+        # Step 2: Create multiprocessing pool to call worker in parallel
+        ################################################################################################################
+        p = Pool(processes=config.pool_size)
+        error_list = p.map(worker, space_list)
+        p.close()
+        p.join()
+
+        for error in error_list:
+            if error is not None and len(error) > 0:
+                logger.error(error)
+
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of outer while
+
+
+########################################################################################################################
+# PROCEDURES:
+#   Step 1: get all combined equipments associated with the space
+#   Step 2: get all equipments associated with the space
+#   Step 3: get all child spaces associated with the space
+#   Step 4: determine start datetime and end datetime to aggregate
+#   Step 5: for each combined equipment in list, get energy output data from energy database
+#   Step 6: for each equipment in list, get energy output data from energy database
+#   Step 7: for each child space in list, get energy output data from energy database
+#   Step 8: determine common time slot to aggregate
+#   Step 9: aggregate energy data in the common time slot by energy categories and hourly
+#   Step 10: save energy data to energy database
+#
+# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter
+########################################################################################################################
+
+def worker(space):
+
+    ####################################################################################################################
+    # Step 1: get all combined equipments associated with the space
+    ####################################################################################################################
+    print("Step 1: get all combined equipments associated with the space")
+
+    cnx_system_db = None
+    cursor_system_db = None
+    try:
+        cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+        cursor_system_db = cnx_system_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 1.1 of space_energy_output_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    combined_equipment_list = list()
+    try:
+        cursor_system_db.execute(" SELECT e.id, e.name "
+                                 " FROM tbl_combined_equipments e, tbl_spaces_combined_equipments se "
+                                 " WHERE e.id = se.combined_equipment_id "
+                                 "       AND e.is_output_counted = true "
+                                 "       AND se.space_id = %s ",
+                                 (space['id'],))
+        rows_combined_equipments = cursor_system_db.fetchall()
+
+        if rows_combined_equipments is not None and len(rows_combined_equipments) > 0:
+            for row in rows_combined_equipments:
+                combined_equipment_list.append({"id": row[0],
+                                                "name": row[1]})
+
+    except Exception as e:
+        error_string = "Error in step 1.2 of space_energy_output_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 2: get all equipments associated with the space
+    ####################################################################################################################
+    print("Step 2: get all equipments associated with the space")
+
+    equipment_list = list()
+    try:
+        cursor_system_db.execute(" SELECT e.id, e.name "
+                                 " FROM tbl_equipments e, tbl_spaces_equipments se "
+                                 " WHERE e.id = se.equipment_id "
+                                 "       AND e.is_output_counted = true "
+                                 "       AND se.space_id = %s ",
+                                 (space['id'],))
+        rows_equipments = cursor_system_db.fetchall()
+
+        if rows_equipments is not None and len(rows_equipments) > 0:
+            for row in rows_equipments:
+                equipment_list.append({"id": row[0],
+                                       "name": row[1]})
+
+    except Exception as e:
+        error_string = "Error in step 2.2 of space_energy_output_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 3: get all child spaces associated with the space
+    ####################################################################################################################
+    print("Step 3: get all child spaces associated with the space")
+
+    child_space_list = list()
+    try:
+        cursor_system_db.execute(" SELECT id, name "
+                                 " FROM tbl_spaces "
+                                 " WHERE is_output_counted = true "
+                                 "       AND parent_space_id = %s ",
+                                 (space['id'],))
+        rows_child_spaces = cursor_system_db.fetchall()
+
+        if rows_child_spaces is not None and len(rows_child_spaces) > 0:
+            for row in rows_child_spaces:
+                child_space_list.append({"id": row[0],
+                                         "name": row[1]})
+
+    except Exception as e:
+        error_string = "Error in step 3 of space_energy_output_category.worker " + str(e)
+        print(error_string)
+        return error_string
+    finally:
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+
+    if ((combined_equipment_list is None or len(combined_equipment_list) == 0) and
+            (equipment_list is None or len(equipment_list) == 0) and
+            (child_space_list is None or len(child_space_list) == 0)):
+        print("This is an empty space ")
+        return None
+
+    ####################################################################################################################
+    # Step 4: determine start datetime and end datetime to aggregate
+    ####################################################################################################################
+    print("Step 4: determine start datetime and end datetime to aggregate")
+    cnx_energy_db = None
+    cursor_energy_db = None
+    try:
+        cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+        cursor_energy_db = cnx_energy_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 4.1 of space_energy_output_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        query = (" SELECT MAX(start_datetime_utc) "
+                 " FROM tbl_space_output_category_hourly "
+                 " WHERE space_id = %s ")
+        cursor_energy_db.execute(query, (space['id'],))
+        row_datetime = cursor_energy_db.fetchone()
+        start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+        start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+        if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+            # replace second and microsecond with 0
+            # note: do not replace minute in case of calculating in half hourly
+            start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+            # start from the next time slot
+            start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+        end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None)
+
+        print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]
+              + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19])
+
+    except Exception as e:
+        error_string = "Error in step 4.2 of space_energy_output_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 5: for each combined equipment in list, get energy output data from energy database
+    ####################################################################################################################
+    energy_combined_equipment_hourly = dict()
+    if combined_equipment_list is not None and len(combined_equipment_list) > 0:
+        try:
+            for combined_equipment in combined_equipment_list:
+                combined_equipment_id = str(combined_equipment['id'])
+                query = (" SELECT start_datetime_utc, energy_category_id, actual_value "
+                         " FROM tbl_combined_equipment_output_category_hourly "
+                         " WHERE combined_equipment_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (combined_equipment_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_combined_equipment_hourly[combined_equipment_id] = None
+                else:
+                    energy_combined_equipment_hourly[combined_equipment_id] = dict()
+                    for row_value in rows_energy_values:
+                        current_datetime_utc = row_value[0]
+                        if current_datetime_utc not in energy_combined_equipment_hourly[combined_equipment_id]:
+                            energy_combined_equipment_hourly[combined_equipment_id][current_datetime_utc] = dict()
+                        energy_category_id = row_value[1]
+                        actual_value = row_value[2]
+                        energy_combined_equipment_hourly[combined_equipment_id][current_datetime_utc][energy_category_id] = \
+                            actual_value
+        except Exception as e:
+            error_string = "Error in step 5 of space_energy_output_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 6: for each equipment in list, get energy output data from energy database
+    ####################################################################################################################
+    energy_equipment_hourly = dict()
+    if equipment_list is not None and len(equipment_list) > 0:
+        try:
+            for equipment in equipment_list:
+                equipment_id = str(equipment['id'])
+                query = (" SELECT start_datetime_utc, energy_category_id, actual_value "
+                         " FROM tbl_equipment_output_category_hourly "
+                         " WHERE equipment_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (equipment_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_equipment_hourly[equipment_id] = None
+                else:
+                    energy_equipment_hourly[equipment_id] = dict()
+                    for row_value in rows_energy_values:
+                        current_datetime_utc = row_value[0]
+                        if current_datetime_utc not in energy_equipment_hourly[equipment_id]:
+                            energy_equipment_hourly[equipment_id][current_datetime_utc] = dict()
+                        energy_category_id = row_value[1]
+                        actual_value = row_value[2]
+                        energy_equipment_hourly[equipment_id][current_datetime_utc][energy_category_id] = \
+                            actual_value
+        except Exception as e:
+            error_string = "Error in step 6 of space_energy_output_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 7: for each child space in list, get energy output data from energy database
+    ####################################################################################################################
+    energy_child_space_hourly = dict()
+    if child_space_list is not None and len(child_space_list) > 0:
+        try:
+            for child_space in child_space_list:
+                child_space_id = str(child_space['id'])
+
+                query = (" SELECT start_datetime_utc, energy_category_id, actual_value "
+                         " FROM tbl_space_output_category_hourly "
+                         " WHERE space_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (child_space_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_child_space_hourly[child_space_id] = None
+                else:
+                    energy_child_space_hourly[child_space_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        current_datetime_utc = row_energy_value[0]
+                        if current_datetime_utc not in energy_child_space_hourly[child_space_id]:
+                            energy_child_space_hourly[child_space_id][current_datetime_utc] = dict()
+                        energy_category_id = row_energy_value[1]
+                        actual_value = row_energy_value[2]
+                        energy_child_space_hourly[child_space_id][current_datetime_utc][energy_category_id] = actual_value
+        except Exception as e:
+            error_string = "Error in step 7 of space_energy_output_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 8: determine common time slot to aggregate
+    ####################################################################################################################
+
+    common_start_datetime_utc = start_datetime_utc
+    common_end_datetime_utc = end_datetime_utc
+
+    print("Getting common time slot of energy values for all combined equipments")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_combined_equipment_hourly is not None and len(energy_combined_equipment_hourly) > 0:
+            for combined_equipment_id, energy_hourly in energy_combined_equipment_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all equipments...")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_equipment_hourly is not None and len(energy_equipment_hourly) > 0:
+            for equipment_id, energy_hourly in energy_equipment_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all child spaces...")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_child_space_hourly is not None and len(energy_child_space_hourly) > 0:
+            for child_space_id, energy_hourly in energy_child_space_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    if (energy_combined_equipment_hourly is None or len(energy_combined_equipment_hourly) == 0) and \
+            (energy_equipment_hourly is None or len(energy_equipment_hourly) == 0) and \
+            (energy_child_space_hourly is None or len(energy_child_space_hourly) == 0):
+        # There isn't any energy data
+        print("There isn't any energy data")
+        # continue the for space loop to the next space
+        print("continue the for space loop to the next space")
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        return None
+
+    print("common_start_datetime_utc: " + str(common_start_datetime_utc))
+    print("common_end_datetime_utc: " + str(common_end_datetime_utc))
+
+    ####################################################################################################################
+    # Step 9: aggregate energy data in the common time slot by energy categories and hourly
+    ####################################################################################################################
+
+    print("Step 9: aggregate energy data in the common time slot by energy categories and hourly")
+    aggregated_values = list()
+    try:
+        current_datetime_utc = common_start_datetime_utc
+        while common_start_datetime_utc is not None \
+                and common_end_datetime_utc is not None \
+                and current_datetime_utc <= common_end_datetime_utc:
+            aggregated_value = dict()
+            aggregated_value['start_datetime_utc'] = current_datetime_utc
+            aggregated_value['meta_data'] = dict()
+
+            if combined_equipment_list is not None and len(combined_equipment_list) > 0:
+                for combined_equipment in combined_equipment_list:
+                    combined_equipment_id = str(combined_equipment['id'])
+                    meta_data_dict = \
+                        energy_combined_equipment_hourly[combined_equipment_id].get(current_datetime_utc, None)
+                    if meta_data_dict is not None and len(meta_data_dict) > 0:
+                        for energy_category_id, actual_value in meta_data_dict.items():
+                            aggregated_value['meta_data'][energy_category_id] = \
+                                aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if equipment_list is not None and len(equipment_list) > 0:
+                for equipment in equipment_list:
+                    equipment_id = str(equipment['id'])
+                    meta_data_dict = energy_equipment_hourly[equipment_id].get(current_datetime_utc, None)
+                    if meta_data_dict is not None and len(meta_data_dict) > 0:
+                        for energy_category_id, actual_value in meta_data_dict.items():
+                            aggregated_value['meta_data'][energy_category_id] = \
+                                aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if child_space_list is not None and len(child_space_list) > 0:
+                for child_space in child_space_list:
+                    child_space_id = str(child_space['id'])
+                    meta_data_dict = energy_child_space_hourly[child_space_id].get(current_datetime_utc, None)
+                    if meta_data_dict is not None and len(meta_data_dict) > 0:
+                        for energy_category_id, actual_value in meta_data_dict.items():
+                            aggregated_value['meta_data'][energy_category_id] = \
+                                aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            aggregated_values.append(aggregated_value)
+
+            current_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+    except Exception as e:
+        error_string = "Error in step 9 of space_energy_output_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 10: save energy data to energy database
+    ####################################################################################################################
+    print("Step 10: save energy data to energy database")
+
+    if len(aggregated_values) > 0:
+        try:
+            add_values = (" INSERT INTO tbl_space_output_category_hourly "
+                          "             (space_id, "
+                          "              energy_category_id, "
+                          "              start_datetime_utc, "
+                          "              actual_value) "
+                          " VALUES  ")
+
+            for aggregated_value in aggregated_values:
+                for energy_category_id, actual_value in aggregated_value['meta_data'].items():
+                    add_values += " (" + str(space['id']) + ","
+                    add_values += " " + str(energy_category_id) + ","
+                    add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "',"
+                    add_values += str(actual_value) + "), "
+            print("add_values:" + add_values)
+            # trim ", " at the end of string and then execute
+            cursor_energy_db.execute(add_values[:-2])
+            cnx_energy_db.commit()
+
+        except Exception as e:
+            error_string = "Error in step 8 of space_energy_output_category.worker " + str(e)
+            print(error_string)
+            return error_string
+        finally:
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+    else:
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()

myems-aggregation/store_billing_input_category.py

@@ -0,0 +1,269 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+import tariff
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all stores
+# for each store in list:
+#   Step 2: get the latest start_datetime_utc
+#   Step 3: get all energy input data since the latest start_datetime_utc
+#   Step 4: get tariffs
+#   Step 5: calculate billing by multiplying energy with tariff
+#   Step 6: save billing data to database
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all stores
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of store_billing_input_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name, cost_center_id "
+                                     " FROM tbl_stores "
+                                     " ORDER BY id ")
+            rows_stores = cursor_system_db.fetchall()
+
+            if rows_stores is None or len(rows_stores) == 0:
+                print("Step 1.2: There isn't any stores. ")
+                if cursor_system_db:
+                    cursor_system_db.close()
+                if cnx_system_db:
+                    cnx_system_db.close()
+                # sleep and continue the outermost while loop
+                time.sleep(60)
+                continue
+
+            store_list = list()
+            for row in rows_stores:
+                store_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of store_billing_input_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Step 1.2: Got all stores from MyEMS System Database")
+
+        cnx_energy_db = None
+        cursor_energy_db = None
+        try:
+            cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+            cursor_energy_db = cnx_energy_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.3 of store_billing_input_category " + str(e))
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Energy Database")
+
+        cnx_billing_db = None
+        cursor_billing_db = None
+        try:
+            cnx_billing_db = mysql.connector.connect(**config.myems_billing_db)
+            cursor_billing_db = cnx_billing_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.4 of store_billing_input_category " + str(e))
+            if cursor_billing_db:
+                cursor_billing_db.close()
+            if cnx_billing_db:
+                cnx_billing_db.close()
+
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Billing Database")
+
+        for store in store_list:
+
+            ############################################################################################################
+            # Step 2: get the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 2: get the latest start_datetime_utc from billing database for " + store['name'])
+            try:
+                cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) "
+                                          " FROM tbl_store_input_category_hourly "
+                                          " WHERE store_id = %s ",
+                                          (store['id'], ))
+                row_datetime = cursor_billing_db.fetchone()
+                start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+                start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+                if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+                    # replace second and microsecond with 0
+                    # note: do not replace minute in case of calculating in half hourly
+                    start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+                    # start from the next time slot
+                    start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+                print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19])
+            except Exception as e:
+                logger.error("Error in step 2 of store_billing_input_category " + str(e))
+                # break the for store loop
+                break
+
+            ############################################################################################################
+            # Step 3: get all energy input data since the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 3: get all energy input data since the latest start_datetime_utc")
+
+            query = (" SELECT start_datetime_utc, energy_category_id, actual_value "
+                     " FROM tbl_store_input_category_hourly "
+                     " WHERE store_id = %s AND start_datetime_utc >= %s "
+                     " ORDER BY id ")
+            cursor_energy_db.execute(query, (store['id'], start_datetime_utc, ))
+            rows_hourly = cursor_energy_db.fetchall()
+
+            if rows_hourly is None or len(rows_hourly) == 0:
+                print("Step 3: There isn't any energy input data to calculate. ")
+                # continue the for store loop
+                continue
+
+            energy_dict = dict()
+            energy_category_list = list()
+            end_datetime_utc = start_datetime_utc
+            for row_hourly in rows_hourly:
+                current_datetime_utc = row_hourly[0]
+                energy_category_id = row_hourly[1]
+
+                if energy_category_id not in energy_category_list:
+                    energy_category_list.append(energy_category_id)
+
+                actual_value = row_hourly[2]
+                if energy_dict.get(current_datetime_utc) is None:
+                    energy_dict[current_datetime_utc] = dict()
+                energy_dict[current_datetime_utc][energy_category_id] = actual_value
+                if current_datetime_utc > end_datetime_utc:
+                    end_datetime_utc = current_datetime_utc
+
+            ############################################################################################################
+            # Step 4: get tariffs
+            ############################################################################################################
+            print("Step 4: get tariffs")
+            tariff_dict = dict()
+            for energy_category_id in energy_category_list:
+                tariff_dict[energy_category_id] = tariff.get_energy_category_tariffs(store['cost_center_id'],
+                                                                                     energy_category_id,
+                                                                                     start_datetime_utc,
+                                                                                     end_datetime_utc)
+            ############################################################################################################
+            # Step 5: calculate billing by multiplying energy with tariff
+            ############################################################################################################
+            print("Step 5: calculate billing by multiplying energy with tariff")
+            billing_dict = dict()
+
+            if len(energy_dict) > 0:
+                for current_datetime_utc in energy_dict.keys():
+                    billing_dict[current_datetime_utc] = dict()
+                    for energy_category_id in energy_category_list:
+                        current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc)
+                        current_energy = energy_dict[current_datetime_utc].get(energy_category_id)
+                        if current_tariff is not None \
+                                and isinstance(current_tariff, Decimal) \
+                                and current_energy is not None \
+                                and isinstance(current_energy, Decimal):
+                            billing_dict[current_datetime_utc][energy_category_id] = \
+                                current_energy * current_tariff
+
+                    if len(billing_dict[current_datetime_utc]) == 0:
+                        del billing_dict[current_datetime_utc]
+
+            ############################################################################################################
+            # Step 6: save billing data to billing database
+            ############################################################################################################
+            print("Step 6: save billing data to billing database")
+
+            if len(billing_dict) > 0:
+                try:
+                    add_values = (" INSERT INTO tbl_store_input_category_hourly "
+                                  "             (store_id, "
+                                  "              energy_category_id, "
+                                  "              start_datetime_utc, "
+                                  "              actual_value) "
+                                  " VALUES  ")
+
+                    for current_datetime_utc in billing_dict:
+                        for energy_category_id in energy_category_list:
+                            current_billing = billing_dict[current_datetime_utc].get(energy_category_id)
+                            if current_billing is not None and isinstance(current_billing, Decimal):
+                                add_values += " (" + str(store['id']) + ","
+                                add_values += " " + str(energy_category_id) + ","
+                                add_values += "'" + current_datetime_utc.isoformat()[0:19] + "',"
+                                add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), "
+                    print("add_values:" + add_values)
+                    # trim ", " at the end of string and then execute
+                    cursor_billing_db.execute(add_values[:-2])
+                    cnx_billing_db.commit()
+                except Exception as e:
+                    logger.error("Error in step 6 of store_billing_input_category " + str(e))
+                    # break the for store loop
+                    break
+
+        # end of for store loop
+        if cnx_system_db:
+            cnx_system_db.close()
+        if cursor_system_db:
+            cursor_system_db.close()
+
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        if cursor_energy_db:
+            cursor_energy_db.close()
+
+        if cnx_billing_db:
+            cnx_billing_db.close()
+        if cursor_billing_db:
+            cursor_billing_db.close()
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of the outermost while loop

myems-aggregation/store_billing_input_item.py

@@ -0,0 +1,269 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+import tariff
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all stores
+# for each store in list:
+#   Step 2: get the latest start_datetime_utc
+#   Step 3: get all energy input data since the latest start_datetime_utc
+#   Step 4: get tariffs
+#   Step 5: calculate billing by multiplying energy with tariff
+#   Step 6: save billing data to database
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all stores
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of store_billing_input_item " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name, cost_center_id "
+                                     " FROM tbl_stores "
+                                     " ORDER BY id ")
+            rows_stores = cursor_system_db.fetchall()
+
+            if rows_stores is None or len(rows_stores) == 0:
+                print("Step 1.2: There isn't any stores. ")
+                if cursor_system_db:
+                    cursor_system_db.close()
+                if cnx_system_db:
+                    cnx_system_db.close()
+                # sleep and continue the outermost while loop
+                time.sleep(60)
+                continue
+
+            store_list = list()
+            for row in rows_stores:
+                store_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of store_billing_input_item " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Step 1.2: Got all stores from MyEMS System Database")
+
+        cnx_energy_db = None
+        cursor_energy_db = None
+        try:
+            cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+            cursor_energy_db = cnx_energy_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.3 of store_billing_input_item " + str(e))
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Energy Database")
+
+        cnx_billing_db = None
+        cursor_billing_db = None
+        try:
+            cnx_billing_db = mysql.connector.connect(**config.myems_billing_db)
+            cursor_billing_db = cnx_billing_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.4 of store_billing_input_item " + str(e))
+            if cursor_billing_db:
+                cursor_billing_db.close()
+            if cnx_billing_db:
+                cnx_billing_db.close()
+
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Billing Database")
+
+        for store in store_list:
+
+            ############################################################################################################
+            # Step 2: get the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 2: get the latest start_datetime_utc from billing database for " + store['name'])
+            try:
+                cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) "
+                                          " FROM tbl_store_input_item_hourly "
+                                          " WHERE store_id = %s ",
+                                          (store['id'], ))
+                row_datetime = cursor_billing_db.fetchone()
+                start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+                start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+                if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+                    # replace second and microsecond with 0
+                    # note: do not replace minute in case of calculating in half hourly
+                    start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+                    # start from the next time slot
+                    start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+                print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19])
+            except Exception as e:
+                logger.error("Error in step 2 of store_billing_input_item " + str(e))
+                # break the for store loop
+                break
+
+            ############################################################################################################
+            # Step 3: get all energy input data since the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 3: get all energy input data since the latest start_datetime_utc")
+
+            query = (" SELECT start_datetime_utc, energy_item_id, actual_value "
+                     " FROM tbl_store_input_item_hourly "
+                     " WHERE store_id = %s AND start_datetime_utc >= %s "
+                     " ORDER BY id ")
+            cursor_energy_db.execute(query, (store['id'], start_datetime_utc, ))
+            rows_hourly = cursor_energy_db.fetchall()
+
+            if rows_hourly is None or len(rows_hourly) == 0:
+                print("Step 3: There isn't any energy input data to calculate. ")
+                # continue the for store loop
+                continue
+
+            energy_dict = dict()
+            energy_item_list = list()
+            end_datetime_utc = start_datetime_utc
+            for row_hourly in rows_hourly:
+                current_datetime_utc = row_hourly[0]
+                energy_item_id = row_hourly[1]
+
+                if energy_item_id not in energy_item_list:
+                    energy_item_list.append(energy_item_id)
+
+                actual_value = row_hourly[2]
+                if energy_dict.get(current_datetime_utc) is None:
+                    energy_dict[current_datetime_utc] = dict()
+                energy_dict[current_datetime_utc][energy_item_id] = actual_value
+                if current_datetime_utc > end_datetime_utc:
+                    end_datetime_utc = current_datetime_utc
+
+            ############################################################################################################
+            # Step 4: get tariffs
+            ############################################################################################################
+            print("Step 4: get tariffs")
+            tariff_dict = dict()
+            for energy_item_id in energy_item_list:
+                tariff_dict[energy_item_id] = tariff.get_energy_item_tariffs(store['cost_center_id'],
+                                                                             energy_item_id,
+                                                                             start_datetime_utc,
+                                                                             end_datetime_utc)
+            ############################################################################################################
+            # Step 5: calculate billing by multiplying energy with tariff
+            ############################################################################################################
+            print("Step 5: calculate billing by multiplying energy with tariff")
+            billing_dict = dict()
+
+            if len(energy_dict) > 0:
+                for current_datetime_utc in energy_dict.keys():
+                    billing_dict[current_datetime_utc] = dict()
+                    for energy_item_id in energy_item_list:
+                        current_tariff = tariff_dict[energy_item_id].get(current_datetime_utc)
+                        current_energy = energy_dict[current_datetime_utc].get(energy_item_id)
+                        if current_tariff is not None \
+                                and isinstance(current_tariff, Decimal) \
+                                and current_energy is not None \
+                                and isinstance(current_energy, Decimal):
+                            billing_dict[current_datetime_utc][energy_item_id] = \
+                                current_energy * current_tariff
+
+                    if len(billing_dict[current_datetime_utc]) == 0:
+                        del billing_dict[current_datetime_utc]
+
+            ############################################################################################################
+            # Step 6: save billing data to billing database
+            ############################################################################################################
+            print("Step 6: save billing data to billing database")
+
+            if len(billing_dict) > 0:
+                try:
+                    add_values = (" INSERT INTO tbl_store_input_item_hourly "
+                                  "             (store_id, "
+                                  "              energy_item_id, "
+                                  "              start_datetime_utc, "
+                                  "              actual_value) "
+                                  " VALUES  ")
+
+                    for current_datetime_utc in billing_dict:
+                        for energy_item_id in energy_item_list:
+                            current_billing = billing_dict[current_datetime_utc].get(energy_item_id)
+                            if current_billing is not None and isinstance(current_billing, Decimal):
+                                add_values += " (" + str(store['id']) + ","
+                                add_values += " " + str(energy_item_id) + ","
+                                add_values += "'" + current_datetime_utc.isoformat()[0:19] + "',"
+                                add_values += str(billing_dict[current_datetime_utc][energy_item_id]) + "), "
+                    print("add_values:" + add_values)
+                    # trim ", " at the end of string and then execute
+                    cursor_billing_db.execute(add_values[:-2])
+                    cnx_billing_db.commit()
+                except Exception as e:
+                    logger.error("Error in step 6 of store_billing_input_item " + str(e))
+                    # break the for store loop
+                    break
+
+        # end of for store loop
+        if cnx_system_db:
+            cnx_system_db.close()
+        if cursor_system_db:
+            cursor_system_db.close()
+
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        if cursor_energy_db:
+            cursor_energy_db.close()
+
+        if cnx_billing_db:
+            cnx_billing_db.close()
+        if cursor_billing_db:
+            cursor_billing_db.close()
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of the outermost while loop

myems-aggregation/store_energy_input_category.py

@@ -0,0 +1,521 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+from multiprocessing import Pool
+import random
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all stores
+# Step 2: Create multiprocessing pool to call worker in parallel
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all stores
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of store_energy_input_category.main " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name "
+                                     " FROM tbl_stores "
+                                     " ORDER BY id ")
+            rows_stores = cursor_system_db.fetchall()
+
+            if rows_stores is None or len(rows_stores) == 0:
+                print("There isn't any stores ")
+                # sleep and continue the outer loop to reconnect the database
+                time.sleep(60)
+                continue
+
+            store_list = list()
+            for row in rows_stores:
+                store_list.append({"id": row[0], "name": row[1]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of store_energy_input_category.main " + str(e))
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        finally:
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+
+        print("Got all stores in MyEMS System Database")
+
+        # shuffle the store list for randomly calculating the meter hourly value
+        random.shuffle(store_list)
+
+        ################################################################################################################
+        # Step 2: Create multiprocessing pool to call worker in parallel
+        ################################################################################################################
+        p = Pool(processes=config.pool_size)
+        error_list = p.map(worker, store_list)
+        p.close()
+        p.join()
+
+        for error in error_list:
+            if error is not None and len(error) > 0:
+                logger.error(error)
+
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of outer while
+
+
+########################################################################################################################
+# PROCEDURES:
+#   Step 1: get all input meters associated with the store
+#   Step 2: get all input virtual meters associated with the store
+#   Step 3: get all input offline meters associated with the store
+#   Step 4: determine start datetime and end datetime to aggregate
+#   Step 5: for each meter in list, get energy input data from energy database
+#   Step 6: for each virtual meter in list, get energy input data from energy database
+#   Step 7: for each offline meter in list, get energy input data from energy database
+#   Step 8: determine common time slot to aggregate
+#   Step 9: aggregate energy data in the common time slot by energy categories and hourly
+#   Step 10: save energy data to energy database
+#
+# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter
+########################################################################################################################
+
+def worker(store):
+    ####################################################################################################################
+    # Step 1: get all input meters associated with the store
+    ####################################################################################################################
+    print("Step 1: get all input meters associated with the store " + str(store['name']))
+
+    meter_list = list()
+    cnx_system_db = None
+    cursor_system_db = None
+    try:
+        cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+        cursor_system_db = cnx_system_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 1.1 of store_energy_input_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_meters m, tbl_stores_meters sm "
+                                 " WHERE m.id = sm.meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND sm.store_id = %s ",
+                                 (store['id'],))
+        rows_meters = cursor_system_db.fetchall()
+
+        if rows_meters is not None and len(rows_meters) > 0:
+            for row in rows_meters:
+                meter_list.append({"id": row[0],
+                                   "name": row[1],
+                                   "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 1.2 of store_energy_input_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 2: get all input virtual meters associated with the store
+    ####################################################################################################################
+    print("Step 2: get all input virtual meters associated with the store")
+    virtual_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_virtual_meters m, tbl_stores_virtual_meters sm "
+                                 " WHERE m.id = sm.virtual_meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND sm.store_id = %s ",
+                                 (store['id'],))
+        rows_virtual_meters = cursor_system_db.fetchall()
+
+        if rows_virtual_meters is not None and len(rows_virtual_meters) > 0:
+            for row in rows_virtual_meters:
+                virtual_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 2.1 of store_energy_input_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 3: get all input offline meters associated with the store
+    ####################################################################################################################
+    print("Step 3: get all input offline meters associated with the store")
+
+    offline_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_offline_meters m, tbl_stores_offline_meters sm "
+                                 " WHERE m.id = sm.offline_meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND sm.store_id = %s ",
+                                 (store['id'],))
+        rows_offline_meters = cursor_system_db.fetchall()
+
+        if rows_offline_meters is not None and len(rows_offline_meters) > 0:
+            for row in rows_offline_meters:
+                offline_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 3.1 of store_energy_input_category.worker " + str(e)
+        print(error_string)
+        return error_string
+    finally:
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+
+    ####################################################################################################################
+    # stop to the next store if this store is empty
+    ####################################################################################################################
+    if (meter_list is None or len(meter_list) == 0) and \
+            (virtual_meter_list is None or len(virtual_meter_list) == 0) and \
+            (offline_meter_list is None or len(offline_meter_list) == 0):
+        print("This is an empty store ")
+        return None
+
+    ####################################################################################################################
+    # Step 4: determine start datetime and end datetime to aggregate
+    ####################################################################################################################
+    print("Step 4: determine start datetime and end datetime to aggregate")
+    cnx_energy_db = None
+    cursor_energy_db = None
+    try:
+        cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+        cursor_energy_db = cnx_energy_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 4.1 of store_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        query = (" SELECT MAX(start_datetime_utc) "
+                 " FROM tbl_store_input_category_hourly "
+                 " WHERE store_id = %s ")
+        cursor_energy_db.execute(query, (store['id'],))
+        row_datetime = cursor_energy_db.fetchone()
+        start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+        start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+        if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+            # replace second and microsecond with 0
+            # note: do not replace minute in case of calculating in half hourly
+            start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+            # start from the next time slot
+            start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+        end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None)
+
+        print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]
+              + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19])
+
+    except Exception as e:
+        error_string = "Error in step 4.2 of store_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 5: for each meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_meter_hourly = dict()
+    try:
+        if meter_list is not None and len(meter_list) > 0:
+            for meter in meter_list:
+                meter_id = str(meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_meter_hourly "
+                         " WHERE meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_meter_hourly[meter_id] = None
+                else:
+                    energy_meter_hourly[meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1]
+    except Exception as e:
+        error_string = "Error in step 5.1 of store_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 6: for each virtual meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_virtual_meter_hourly = dict()
+    if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+        try:
+            for virtual_meter in virtual_meter_list:
+                virtual_meter_id = str(virtual_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_virtual_meter_hourly "
+                         " WHERE virtual_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_virtual_meter_hourly[virtual_meter_id] = None
+                else:
+                    energy_virtual_meter_hourly[virtual_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1]
+        except Exception as e:
+            error_string = "Error in step 6.1 of store_energy_input_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 7: for each offline meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_offline_meter_hourly = dict()
+    if offline_meter_list is not None and len(offline_meter_list) > 0:
+        try:
+            for offline_meter in offline_meter_list:
+                offline_meter_id = str(offline_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_offline_meter_hourly "
+                         " WHERE offline_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_offline_meter_hourly[offline_meter_id] = None
+                else:
+                    energy_offline_meter_hourly[offline_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1]
+
+        except Exception as e:
+            error_string = "Error in step 7.1 of store_energy_input_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 8: determine common time slot to aggregate
+    ####################################################################################################################
+
+    common_start_datetime_utc = start_datetime_utc
+    common_end_datetime_utc = end_datetime_utc
+
+    print("Getting common time slot of energy values for all meters")
+    if energy_meter_hourly is not None and len(energy_meter_hourly) > 0:
+        for meter_id, energy_hourly in energy_meter_hourly.items():
+            if energy_hourly is None or len(energy_hourly) == 0:
+                common_start_datetime_utc = None
+                common_end_datetime_utc = None
+                break
+            else:
+                if common_start_datetime_utc < min(energy_hourly.keys()):
+                    common_start_datetime_utc = min(energy_hourly.keys())
+                if common_end_datetime_utc > max(energy_hourly.keys()):
+                    common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all virtual meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_virtual_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all offline meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_offline_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \
+            (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \
+            (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0):
+        # There isn't any energy data
+        print("There isn't any energy data")
+        # continue the for store loop to the next store
+        print("continue the for store loop to the next store")
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        return None
+
+    print("common_start_datetime_utc: " + str(common_start_datetime_utc))
+    print("common_end_datetime_utc: " + str(common_end_datetime_utc))
+
+    ####################################################################################################################
+    # Step 9: aggregate energy data in the common time slot by energy categories and hourly
+    ####################################################################################################################
+
+    print("Step 9: aggregate energy data in the common time slot by energy categories and hourly")
+    aggregated_values = list()
+    try:
+        current_datetime_utc = common_start_datetime_utc
+        while common_start_datetime_utc is not None \
+                and common_end_datetime_utc is not None \
+                and current_datetime_utc <= common_end_datetime_utc:
+            aggregated_value = dict()
+            aggregated_value['start_datetime_utc'] = current_datetime_utc
+            aggregated_value['meta_data'] = dict()
+
+            if meter_list is not None and len(meter_list) > 0:
+                for meter in meter_list:
+                    meter_id = str(meter['id'])
+                    energy_category_id = meter['energy_category_id']
+                    actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+                for virtual_meter in virtual_meter_list:
+                    virtual_meter_id = str(virtual_meter['id'])
+                    energy_category_id = virtual_meter['energy_category_id']
+                    actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if offline_meter_list is not None and len(offline_meter_list) > 0:
+                for offline_meter in offline_meter_list:
+                    offline_meter_id = str(offline_meter['id'])
+                    energy_category_id = offline_meter['energy_category_id']
+                    actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            aggregated_values.append(aggregated_value)
+
+            current_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+    except Exception as e:
+        error_string = "Error in step 9 of store_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 10: save energy data to energy database
+    ####################################################################################################################
+    print("Step 10: save energy data to energy database")
+
+    if len(aggregated_values) > 0:
+        try:
+            add_values = (" INSERT INTO tbl_store_input_category_hourly "
+                          "             (store_id, "
+                          "              energy_category_id, "
+                          "              start_datetime_utc, "
+                          "              actual_value) "
+                          " VALUES  ")
+
+            for aggregated_value in aggregated_values:
+                for energy_category_id, actual_value in aggregated_value['meta_data'].items():
+                    add_values += " (" + str(store['id']) + ","
+                    add_values += " " + str(energy_category_id) + ","
+                    add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "',"
+                    add_values += str(actual_value) + "), "
+            print("add_values:" + add_values)
+            # trim ", " at the end of string and then execute
+            cursor_energy_db.execute(add_values[:-2])
+            cnx_energy_db.commit()
+
+        except Exception as e:
+            error_string = "Error in step 10.1 of store_energy_input_category.worker " + str(e)
+            print(error_string)
+            return error_string
+        finally:
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+    else:
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()

myems-aggregation/store_energy_input_item.py

@@ -0,0 +1,524 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+from multiprocessing import Pool
+import random
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all stores
+# Step 2: Create multiprocessing pool to call worker in parallel
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all stores
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of store_energy_input_item.main " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name "
+                                     " FROM tbl_stores "
+                                     " ORDER BY id ")
+            rows_stores = cursor_system_db.fetchall()
+
+            if rows_stores is None or len(rows_stores) == 0:
+                print("There isn't any stores ")
+                # sleep and continue the outer loop to reconnect the database
+                time.sleep(60)
+                continue
+
+            store_list = list()
+            for row in rows_stores:
+                store_list.append({"id": row[0], "name": row[1]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of store_energy_input_item.main " + str(e))
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        finally:
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+
+        print("Got all stores in MyEMS System Database")
+
+        # shuffle the store list for randomly calculating the meter hourly value
+        random.shuffle(store_list)
+
+        ################################################################################################################
+        # Step 2: Create multiprocessing pool to call worker in parallel
+        ################################################################################################################
+        p = Pool(processes=config.pool_size)
+        error_list = p.map(worker, store_list)
+        p.close()
+        p.join()
+
+        for error in error_list:
+            if error is not None and len(error) > 0:
+                logger.error(error)
+
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of outer while
+
+
+########################################################################################################################
+# PROCEDURES:
+#   Step 1: get all input meters associated with the store
+#   Step 2: get all input virtual meters associated with the store
+#   Step 3: get all input offline meters associated with the store
+#   Step 4: determine start datetime and end datetime to aggregate
+#   Step 5: for each meter in list, get energy input data from energy database
+#   Step 6: for each virtual meter in list, get energy input data from energy database
+#   Step 7: for each offline meter in list, get energy input data from energy database
+#   Step 8: determine common time slot to aggregate
+#   Step 9: aggregate energy data in the common time slot by energy items and hourly
+#   Step 10: save energy data to energy database
+#
+# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter
+########################################################################################################################
+
+def worker(store):
+    ####################################################################################################################
+    # Step 1: get all input meters associated with the store
+    ####################################################################################################################
+    print("Step 1: get all input meters associated with the store " + str(store['name']))
+
+    meter_list = list()
+    cnx_system_db = None
+    cursor_system_db = None
+    try:
+        cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+        cursor_system_db = cnx_system_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 1.1 of store_energy_input_item.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id "
+                                 " FROM tbl_meters m, tbl_stores_meters sm "
+                                 " WHERE m.id = sm.meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND m.energy_item_id is NOT NULL "
+                                 "       AND sm.store_id = %s ",
+                                 (store['id'],))
+        rows_meters = cursor_system_db.fetchall()
+
+        if rows_meters is not None and len(rows_meters) > 0:
+            for row in rows_meters:
+                meter_list.append({"id": row[0],
+                                   "name": row[1],
+                                   "energy_item_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 1.2 of store_energy_input_item.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 2: get all input virtual meters associated with the store
+    ####################################################################################################################
+    print("Step 2: get all input virtual meters associated with the store")
+    virtual_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id "
+                                 " FROM tbl_virtual_meters m, tbl_stores_virtual_meters sm "
+                                 " WHERE m.id = sm.virtual_meter_id "
+                                 "       AND m.energy_item_id is NOT NULL "
+                                 "       AND m.is_counted = true "
+                                 "       AND sm.store_id = %s ",
+                                 (store['id'],))
+        rows_virtual_meters = cursor_system_db.fetchall()
+
+        if rows_virtual_meters is not None and len(rows_virtual_meters) > 0:
+            for row in rows_virtual_meters:
+                virtual_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_item_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 2.1 of store_energy_input_item.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 3: get all input offline meters associated with the store
+    ####################################################################################################################
+    print("Step 3: get all input offline meters associated with the store")
+
+    offline_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id "
+                                 " FROM tbl_offline_meters m, tbl_stores_offline_meters sm "
+                                 " WHERE m.id = sm.offline_meter_id "
+                                 "       AND m.energy_item_id is NOT NULL "
+                                 "       AND m.is_counted = true "
+                                 "       AND sm.store_id = %s ",
+                                 (store['id'],))
+        rows_offline_meters = cursor_system_db.fetchall()
+
+        if rows_offline_meters is not None and len(rows_offline_meters) > 0:
+            for row in rows_offline_meters:
+                offline_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_item_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 3.1 of store_energy_input_item.worker " + str(e)
+        print(error_string)
+        return error_string
+    finally:
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+
+    ####################################################################################################################
+    # stop to the next store if this store is empty
+    ####################################################################################################################
+    if (meter_list is None or len(meter_list) == 0) and \
+            (virtual_meter_list is None or len(virtual_meter_list) == 0) and \
+            (offline_meter_list is None or len(offline_meter_list) == 0):
+        print("This is an empty store ")
+        return None
+
+    ####################################################################################################################
+    # Step 4: determine start datetime and end datetime to aggregate
+    ####################################################################################################################
+    print("Step 4: determine start datetime and end datetime to aggregate")
+    cnx_energy_db = None
+    cursor_energy_db = None
+    try:
+        cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+        cursor_energy_db = cnx_energy_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 4.1 of store_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        query = (" SELECT MAX(start_datetime_utc) "
+                 " FROM tbl_store_input_item_hourly "
+                 " WHERE store_id = %s ")
+        cursor_energy_db.execute(query, (store['id'],))
+        row_datetime = cursor_energy_db.fetchone()
+        start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+        start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+        if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+            # replace second and microsecond with 0
+            # note: do not replace minute in case of calculating in half hourly
+            start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+            # start from the next time slot
+            start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+        end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None)
+
+        print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]
+              + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19])
+
+    except Exception as e:
+        error_string = "Error in step 4.2 of store_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 5: for each meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_meter_hourly = dict()
+    try:
+        if meter_list is not None and len(meter_list) > 0:
+            for meter in meter_list:
+                meter_id = str(meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_meter_hourly "
+                         " WHERE meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_meter_hourly[meter_id] = None
+                else:
+                    energy_meter_hourly[meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1]
+    except Exception as e:
+        error_string = "Error in step 5.1 of store_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 6: for each virtual meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_virtual_meter_hourly = dict()
+    if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+        try:
+            for virtual_meter in virtual_meter_list:
+                virtual_meter_id = str(virtual_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_virtual_meter_hourly "
+                         " WHERE virtual_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_virtual_meter_hourly[virtual_meter_id] = None
+                else:
+                    energy_virtual_meter_hourly[virtual_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1]
+        except Exception as e:
+            error_string = "Error in step 6.1 of store_energy_input_item.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 7: for each offline meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_offline_meter_hourly = dict()
+    if offline_meter_list is not None and len(offline_meter_list) > 0:
+        try:
+            for offline_meter in offline_meter_list:
+                offline_meter_id = str(offline_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_offline_meter_hourly "
+                         " WHERE offline_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_offline_meter_hourly[offline_meter_id] = None
+                else:
+                    energy_offline_meter_hourly[offline_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1]
+
+        except Exception as e:
+            error_string = "Error in step 7.1 of store_energy_input_item.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 8: determine common time slot to aggregate
+    ####################################################################################################################
+
+    common_start_datetime_utc = start_datetime_utc
+    common_end_datetime_utc = end_datetime_utc
+
+    print("Getting common time slot of energy values for all meters")
+    if energy_meter_hourly is not None and len(energy_meter_hourly) > 0:
+        for meter_id, energy_hourly in energy_meter_hourly.items():
+            if energy_hourly is None or len(energy_hourly) == 0:
+                common_start_datetime_utc = None
+                common_end_datetime_utc = None
+                break
+            else:
+                if common_start_datetime_utc < min(energy_hourly.keys()):
+                    common_start_datetime_utc = min(energy_hourly.keys())
+                if common_end_datetime_utc > max(energy_hourly.keys()):
+                    common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all virtual meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_virtual_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all offline meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_offline_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \
+            (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \
+            (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0):
+        # There isn't any energy data
+        print("There isn't any energy data")
+        # continue the for store loop to the next store
+        print("continue the for store loop to the next store")
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        return None
+
+    print("common_start_datetime_utc: " + str(common_start_datetime_utc))
+    print("common_end_datetime_utc: " + str(common_end_datetime_utc))
+
+    ####################################################################################################################
+    # Step 9: aggregate energy data in the common time slot by energy items and hourly
+    ####################################################################################################################
+
+    print("Step 9: aggregate energy data in the common time slot by energy items and hourly")
+    aggregated_values = list()
+    try:
+        current_datetime_utc = common_start_datetime_utc
+        while common_start_datetime_utc is not None \
+                and common_end_datetime_utc is not None \
+                and current_datetime_utc <= common_end_datetime_utc:
+            aggregated_value = dict()
+            aggregated_value['start_datetime_utc'] = current_datetime_utc
+            aggregated_value['meta_data'] = dict()
+
+            if meter_list is not None and len(meter_list) > 0:
+                for meter in meter_list:
+                    meter_id = str(meter['id'])
+                    energy_item_id = meter['energy_item_id']
+                    actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_item_id] = \
+                        aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value
+
+            if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+                for virtual_meter in virtual_meter_list:
+                    virtual_meter_id = str(virtual_meter['id'])
+                    energy_item_id = virtual_meter['energy_item_id']
+                    actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_item_id] = \
+                        aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value
+
+            if offline_meter_list is not None and len(offline_meter_list) > 0:
+                for offline_meter in offline_meter_list:
+                    offline_meter_id = str(offline_meter['id'])
+                    energy_item_id = offline_meter['energy_item_id']
+                    actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_item_id] = \
+                        aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value
+
+            aggregated_values.append(aggregated_value)
+
+            current_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+    except Exception as e:
+        error_string = "Error in step 9 of store_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 10: save energy data to energy database
+    ####################################################################################################################
+    print("Step 10: save energy data to energy database")
+
+    if len(aggregated_values) > 0:
+        try:
+            add_values = (" INSERT INTO tbl_store_input_item_hourly "
+                          "             (store_id, "
+                          "              energy_item_id, "
+                          "              start_datetime_utc, "
+                          "              actual_value) "
+                          " VALUES  ")
+
+            for aggregated_value in aggregated_values:
+                for energy_item_id, actual_value in aggregated_value['meta_data'].items():
+                    add_values += " (" + str(store['id']) + ","
+                    add_values += " " + str(energy_item_id) + ","
+                    add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "',"
+                    add_values += str(actual_value) + "), "
+            print("add_values:" + add_values)
+            # trim ", " at the end of string and then execute
+            cursor_energy_db.execute(add_values[:-2])
+            cnx_energy_db.commit()
+
+        except Exception as e:
+            error_string = "Error in step 10.1 of store_energy_input_item.worker " + str(e)
+            print(error_string)
+            return error_string
+        finally:
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+    else:
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()

myems-aggregation/tariff.py

@@ -0,0 +1,205 @@
+from datetime import timedelta
+import mysql.connector
+import config
+import collections
+
+
+########################################################################################################################
+# Get tariffs by energy category
+########################################################################################################################
+def get_energy_category_tariffs(cost_center_id, energy_category_id, start_datetime_utc, end_datetime_utc):
+    # todo: verify parameters
+    if cost_center_id is None:
+        return dict()
+
+    # get timezone offset in minutes, this value will be returned to client
+    timezone_offset = int(config.utc_offset[1:3]) * 60 + int(config.utc_offset[4:6])
+    if config.utc_offset[0] == '-':
+        timezone_offset = -timezone_offset
+
+    tariff_dict = collections.OrderedDict()
+
+    cnx = None
+    cursor = None
+    try:
+        cnx = mysql.connector.connect(**config.myems_system_db)
+        cursor = cnx.cursor()
+        query_tariffs = (" SELECT t.id, t.valid_from_datetime_utc, t.valid_through_datetime_utc "
+                         " FROM tbl_tariffs t, tbl_cost_centers_tariffs cct "
+                         " WHERE t.energy_category_id = %s AND "
+                         "       t.id = cct.tariff_id AND "
+                         "       cct.cost_center_id = %s AND "
+                         "       t.valid_through_datetime_utc >= %s AND "
+                         "       t.valid_from_datetime_utc <= %s "
+                         " ORDER BY t.valid_from_datetime_utc ")
+        cursor.execute(query_tariffs, (energy_category_id, cost_center_id, start_datetime_utc, end_datetime_utc,))
+        rows_tariffs = cursor.fetchall()
+    except Exception as e:
+        print(str(e))
+        if cnx:
+            cnx.disconnect()
+        if cursor:
+            cursor.close()
+        return dict()
+
+    if rows_tariffs is None or len(rows_tariffs) == 0:
+        if cursor:
+            cursor.close()
+        if cnx:
+            cnx.disconnect()
+        return dict()
+
+    for row in rows_tariffs:
+        tariff_dict[row[0]] = {'valid_from_datetime_utc': row[1],
+                               'valid_through_datetime_utc': row[2],
+                               'rates': list()}
+
+    try:
+        query_timeofuse_tariffs = (" SELECT tariff_id, start_time_of_day, end_time_of_day, price "
+                                   " FROM tbl_tariffs_timeofuses "
+                                   " WHERE tariff_id IN ( " + ', '.join(map(str, tariff_dict.keys())) + ")"
+                                   " ORDER BY tariff_id, start_time_of_day ")
+        cursor.execute(query_timeofuse_tariffs, )
+        rows_timeofuse_tariffs = cursor.fetchall()
+    except Exception as e:
+        print(str(e))
+        if cnx:
+            cnx.disconnect()
+        if cursor:
+            cursor.close()
+        return dict()
+
+    if cursor:
+        cursor.close()
+    if cnx:
+        cnx.disconnect()
+
+    if rows_timeofuse_tariffs is None or len(rows_timeofuse_tariffs) == 0:
+        return dict()
+
+    for row in rows_timeofuse_tariffs:
+        tariff_dict[row[0]]['rates'].append({'start_time_of_day': row[1],
+                                             'end_time_of_day': row[2],
+                                             'price': row[3]})
+
+    result = dict()
+    for tariff_id, tariff_value in tariff_dict.items():
+        current_datetime_utc = tariff_value['valid_from_datetime_utc']
+        while current_datetime_utc < tariff_value['valid_through_datetime_utc']:
+            for rate in tariff_value['rates']:
+                current_datetime_local = current_datetime_utc + timedelta(minutes=timezone_offset)
+                seconds_since_midnight = (current_datetime_local -
+                                          current_datetime_local.replace(hour=0,
+                                                                         second=0,
+                                                                         microsecond=0,
+                                                                         tzinfo=None)).total_seconds()
+                if rate['start_time_of_day'].total_seconds() <= \
+                        seconds_since_midnight < rate['end_time_of_day'].total_seconds():
+                    result[current_datetime_utc] = rate['price']
+                    break
+
+            # start from the next time slot
+            current_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+    return {k: v for k, v in result.items() if start_datetime_utc <= k <= end_datetime_utc}
+
+
+########################################################################################################################
+# Get tariffs by energy item
+########################################################################################################################
+def get_energy_item_tariffs(cost_center_id, energy_item_id, start_datetime_utc, end_datetime_utc):
+    # todo: verify parameters
+    if cost_center_id is None:
+        return dict()
+
+    # get timezone offset in minutes, this value will be returned to client
+    timezone_offset = int(config.utc_offset[1:3]) * 60 + int(config.utc_offset[4:6])
+    if config.utc_offset[0] == '-':
+        timezone_offset = -timezone_offset
+
+    tariff_dict = collections.OrderedDict()
+
+    cnx = None
+    cursor = None
+    try:
+        cnx = mysql.connector.connect(**config.myems_system_db)
+        cursor = cnx.cursor()
+        query_tariffs = (" SELECT t.id, t.valid_from_datetime_utc, t.valid_through_datetime_utc "
+                         " FROM tbl_tariffs t, tbl_cost_centers_tariffs cct, tbl_energy_items ei "
+                         " WHERE ei.id = %s AND "
+                         "       t.energy_category_id = ei.energy_category_id AND "
+                         "       t.id = cct.tariff_id AND "
+                         "       cct.cost_center_id = %s AND "
+                         "       t.valid_through_datetime_utc >= %s AND "
+                         "       t.valid_from_datetime_utc <= %s "
+                         " ORDER BY t.valid_from_datetime_utc ")
+        cursor.execute(query_tariffs, (energy_item_id, cost_center_id, start_datetime_utc, end_datetime_utc,))
+        rows_tariffs = cursor.fetchall()
+    except Exception as e:
+        print(str(e))
+        if cnx:
+            cnx.disconnect()
+        if cursor:
+            cursor.close()
+        return dict()
+
+    if rows_tariffs is None or len(rows_tariffs) == 0:
+        if cursor:
+            cursor.close()
+        if cnx:
+            cnx.disconnect()
+        return dict()
+
+    for row in rows_tariffs:
+        tariff_dict[row[0]] = {'valid_from_datetime_utc': row[1],
+                               'valid_through_datetime_utc': row[2],
+                               'rates': list()}
+
+    try:
+        query_timeofuse_tariffs = (" SELECT tariff_id, start_time_of_day, end_time_of_day, price "
+                                   " FROM tbl_tariffs_timeofuses "
+                                   " WHERE tariff_id IN ( " + ', '.join(map(str, tariff_dict.keys())) + ")"
+                                   " ORDER BY tariff_id, start_time_of_day ")
+        cursor.execute(query_timeofuse_tariffs, )
+        rows_timeofuse_tariffs = cursor.fetchall()
+    except Exception as e:
+        print(str(e))
+        if cnx:
+            cnx.disconnect()
+        if cursor:
+            cursor.close()
+        return dict()
+
+    if cursor:
+        cursor.close()
+    if cnx:
+        cnx.disconnect()
+
+    if rows_timeofuse_tariffs is None or len(rows_timeofuse_tariffs) == 0:
+        return dict()
+
+    for row in rows_timeofuse_tariffs:
+        tariff_dict[row[0]]['rates'].append({'start_time_of_day': row[1],
+                                             'end_time_of_day': row[2],
+                                             'price': row[3]})
+
+    result = dict()
+    for tariff_id, tariff_value in tariff_dict.items():
+        current_datetime_utc = tariff_value['valid_from_datetime_utc']
+        while current_datetime_utc < tariff_value['valid_through_datetime_utc']:
+            for rate in tariff_value['rates']:
+                current_datetime_local = current_datetime_utc + timedelta(minutes=timezone_offset)
+                seconds_since_midnight = (current_datetime_local -
+                                          current_datetime_local.replace(hour=0,
+                                                                         second=0,
+                                                                         microsecond=0,
+                                                                         tzinfo=None)).total_seconds()
+                if rate['start_time_of_day'].total_seconds() <= \
+                        seconds_since_midnight < rate['end_time_of_day'].total_seconds():
+                    result[current_datetime_utc] = rate['price']
+                    break
+
+            # start from the next time slot
+            current_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+    return {k: v for k, v in result.items() if start_datetime_utc <= k <= end_datetime_utc}

myems-aggregation/tenant_billing_input_category.py

@@ -0,0 +1,269 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+import tariff
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all tenants
+# for each tenant in list:
+#   Step 2: get the latest start_datetime_utc
+#   Step 3: get all energy input data since the latest start_datetime_utc
+#   Step 4: get tariffs
+#   Step 5: calculate billing by multiplying energy with tariff
+#   Step 6: save billing data to database
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all tenants
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of tenant_billing_input_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name, cost_center_id "
+                                     " FROM tbl_tenants "
+                                     " ORDER BY id ")
+            rows_tenants = cursor_system_db.fetchall()
+
+            if rows_tenants is None or len(rows_tenants) == 0:
+                print("Step 1.2: There isn't any tenants. ")
+                if cursor_system_db:
+                    cursor_system_db.close()
+                if cnx_system_db:
+                    cnx_system_db.close()
+                # sleep and continue the outermost while loop
+                time.sleep(60)
+                continue
+
+            tenant_list = list()
+            for row in rows_tenants:
+                tenant_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of tenant_billing_input_category " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Step 1.2: Got all tenants from MyEMS System Database")
+
+        cnx_energy_db = None
+        cursor_energy_db = None
+        try:
+            cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+            cursor_energy_db = cnx_energy_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.3 of tenant_billing_input_category " + str(e))
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Energy Database")
+
+        cnx_billing_db = None
+        cursor_billing_db = None
+        try:
+            cnx_billing_db = mysql.connector.connect(**config.myems_billing_db)
+            cursor_billing_db = cnx_billing_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.4 of tenant_billing_input_category " + str(e))
+            if cursor_billing_db:
+                cursor_billing_db.close()
+            if cnx_billing_db:
+                cnx_billing_db.close()
+
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Billing Database")
+
+        for tenant in tenant_list:
+
+            ############################################################################################################
+            # Step 2: get the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 2: get the latest start_datetime_utc from billing database for " + tenant['name'])
+            try:
+                cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) "
+                                          " FROM tbl_tenant_input_category_hourly "
+                                          " WHERE tenant_id = %s ",
+                                          (tenant['id'], ))
+                row_datetime = cursor_billing_db.fetchone()
+                start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+                start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+                if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+                    # replace second and microsecond with 0
+                    # note: do not replace minute in case of calculating in half hourly
+                    start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+                    # start from the next time slot
+                    start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+                print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19])
+            except Exception as e:
+                logger.error("Error in step 2 of tenant_billing_input_category " + str(e))
+                # break the for tenant loop
+                break
+
+            ############################################################################################################
+            # Step 3: get all energy input data since the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 3: get all energy input data since the latest start_datetime_utc")
+
+            query = (" SELECT start_datetime_utc, energy_category_id, actual_value "
+                     " FROM tbl_tenant_input_category_hourly "
+                     " WHERE tenant_id = %s AND start_datetime_utc >= %s "
+                     " ORDER BY id ")
+            cursor_energy_db.execute(query, (tenant['id'], start_datetime_utc, ))
+            rows_hourly = cursor_energy_db.fetchall()
+
+            if rows_hourly is None or len(rows_hourly) == 0:
+                print("Step 3: There isn't any energy input data to calculate. ")
+                # continue the for tenant loop
+                continue
+
+            energy_dict = dict()
+            energy_category_list = list()
+            end_datetime_utc = start_datetime_utc
+            for row_hourly in rows_hourly:
+                current_datetime_utc = row_hourly[0]
+                energy_category_id = row_hourly[1]
+
+                if energy_category_id not in energy_category_list:
+                    energy_category_list.append(energy_category_id)
+
+                actual_value = row_hourly[2]
+                if energy_dict.get(current_datetime_utc) is None:
+                    energy_dict[current_datetime_utc] = dict()
+                energy_dict[current_datetime_utc][energy_category_id] = actual_value
+                if current_datetime_utc > end_datetime_utc:
+                    end_datetime_utc = current_datetime_utc
+
+            ############################################################################################################
+            # Step 4: get tariffs
+            ############################################################################################################
+            print("Step 4: get tariffs")
+            tariff_dict = dict()
+            for energy_category_id in energy_category_list:
+                tariff_dict[energy_category_id] = tariff.get_energy_category_tariffs(tenant['cost_center_id'],
+                                                                                     energy_category_id,
+                                                                                     start_datetime_utc,
+                                                                                     end_datetime_utc)
+            ############################################################################################################
+            # Step 5: calculate billing by multiplying energy with tariff
+            ############################################################################################################
+            print("Step 5: calculate billing by multiplying energy with tariff")
+            billing_dict = dict()
+
+            if len(energy_dict) > 0:
+                for current_datetime_utc in energy_dict.keys():
+                    billing_dict[current_datetime_utc] = dict()
+                    for energy_category_id in energy_category_list:
+                        current_tariff = tariff_dict[energy_category_id].get(current_datetime_utc)
+                        current_energy = energy_dict[current_datetime_utc].get(energy_category_id)
+                        if current_tariff is not None \
+                                and isinstance(current_tariff, Decimal) \
+                                and current_energy is not None \
+                                and isinstance(current_energy, Decimal):
+                            billing_dict[current_datetime_utc][energy_category_id] = \
+                                current_energy * current_tariff
+
+                    if len(billing_dict[current_datetime_utc]) == 0:
+                        del billing_dict[current_datetime_utc]
+
+            ############################################################################################################
+            # Step 6: save billing data to billing database
+            ############################################################################################################
+            print("Step 6: save billing data to billing database")
+
+            if len(billing_dict) > 0:
+                try:
+                    add_values = (" INSERT INTO tbl_tenant_input_category_hourly "
+                                  "             (tenant_id, "
+                                  "              energy_category_id, "
+                                  "              start_datetime_utc, "
+                                  "              actual_value) "
+                                  " VALUES  ")
+
+                    for current_datetime_utc in billing_dict:
+                        for energy_category_id in energy_category_list:
+                            current_billing = billing_dict[current_datetime_utc].get(energy_category_id)
+                            if current_billing is not None and isinstance(current_billing, Decimal):
+                                add_values += " (" + str(tenant['id']) + ","
+                                add_values += " " + str(energy_category_id) + ","
+                                add_values += "'" + current_datetime_utc.isoformat()[0:19] + "',"
+                                add_values += str(billing_dict[current_datetime_utc][energy_category_id]) + "), "
+                    print("add_values:" + add_values)
+                    # trim ", " at the end of string and then execute
+                    cursor_billing_db.execute(add_values[:-2])
+                    cnx_billing_db.commit()
+                except Exception as e:
+                    logger.error("Error in step 6 of tenant_billing_input_category " + str(e))
+                    # break the for tenant loop
+                    break
+
+        # end of for tenant loop
+        if cnx_system_db:
+            cnx_system_db.close()
+        if cursor_system_db:
+            cursor_system_db.close()
+
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        if cursor_energy_db:
+            cursor_energy_db.close()
+
+        if cnx_billing_db:
+            cnx_billing_db.close()
+        if cursor_billing_db:
+            cursor_billing_db.close()
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of the outermost while loop

myems-aggregation/tenant_billing_input_item.py

@@ -0,0 +1,269 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+import tariff
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all tenants
+# for each tenant in list:
+#   Step 2: get the latest start_datetime_utc
+#   Step 3: get all energy input data since the latest start_datetime_utc
+#   Step 4: get tariffs
+#   Step 5: calculate billing by multiplying energy with tariff
+#   Step 6: save billing data to database
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all tenants
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of tenant_billing_input_item " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name, cost_center_id "
+                                     " FROM tbl_tenants "
+                                     " ORDER BY id ")
+            rows_tenants = cursor_system_db.fetchall()
+
+            if rows_tenants is None or len(rows_tenants) == 0:
+                print("Step 1.2: There isn't any tenants. ")
+                if cursor_system_db:
+                    cursor_system_db.close()
+                if cnx_system_db:
+                    cnx_system_db.close()
+                # sleep and continue the outermost while loop
+                time.sleep(60)
+                continue
+
+            tenant_list = list()
+            for row in rows_tenants:
+                tenant_list.append({"id": row[0], "name": row[1], "cost_center_id": row[2]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of tenant_billing_input_item " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Step 1.2: Got all tenants from MyEMS System Database")
+
+        cnx_energy_db = None
+        cursor_energy_db = None
+        try:
+            cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+            cursor_energy_db = cnx_energy_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.3 of tenant_billing_input_item " + str(e))
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Energy Database")
+
+        cnx_billing_db = None
+        cursor_billing_db = None
+        try:
+            cnx_billing_db = mysql.connector.connect(**config.myems_billing_db)
+            cursor_billing_db = cnx_billing_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.4 of tenant_billing_input_item " + str(e))
+            if cursor_billing_db:
+                cursor_billing_db.close()
+            if cnx_billing_db:
+                cnx_billing_db.close()
+
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outermost while loop
+            time.sleep(60)
+            continue
+
+        print("Connected to MyEMS Billing Database")
+
+        for tenant in tenant_list:
+
+            ############################################################################################################
+            # Step 2: get the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 2: get the latest start_datetime_utc from billing database for " + tenant['name'])
+            try:
+                cursor_billing_db.execute(" SELECT MAX(start_datetime_utc) "
+                                          " FROM tbl_tenant_input_item_hourly "
+                                          " WHERE tenant_id = %s ",
+                                          (tenant['id'], ))
+                row_datetime = cursor_billing_db.fetchone()
+                start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+                start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+                if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+                    # replace second and microsecond with 0
+                    # note: do not replace minute in case of calculating in half hourly
+                    start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+                    # start from the next time slot
+                    start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+                print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19])
+            except Exception as e:
+                logger.error("Error in step 2 of tenant_billing_input_item " + str(e))
+                # break the for tenant loop
+                break
+
+            ############################################################################################################
+            # Step 3: get all energy input data since the latest start_datetime_utc
+            ############################################################################################################
+            print("Step 3: get all energy input data since the latest start_datetime_utc")
+
+            query = (" SELECT start_datetime_utc, energy_item_id, actual_value "
+                     " FROM tbl_tenant_input_item_hourly "
+                     " WHERE tenant_id = %s AND start_datetime_utc >= %s "
+                     " ORDER BY id ")
+            cursor_energy_db.execute(query, (tenant['id'], start_datetime_utc, ))
+            rows_hourly = cursor_energy_db.fetchall()
+
+            if rows_hourly is None or len(rows_hourly) == 0:
+                print("Step 3: There isn't any energy input data to calculate. ")
+                # continue the for tenant loop
+                continue
+
+            energy_dict = dict()
+            energy_item_list = list()
+            end_datetime_utc = start_datetime_utc
+            for row_hourly in rows_hourly:
+                current_datetime_utc = row_hourly[0]
+                energy_item_id = row_hourly[1]
+
+                if energy_item_id not in energy_item_list:
+                    energy_item_list.append(energy_item_id)
+
+                actual_value = row_hourly[2]
+                if energy_dict.get(current_datetime_utc) is None:
+                    energy_dict[current_datetime_utc] = dict()
+                energy_dict[current_datetime_utc][energy_item_id] = actual_value
+                if current_datetime_utc > end_datetime_utc:
+                    end_datetime_utc = current_datetime_utc
+
+            ############################################################################################################
+            # Step 4: get tariffs
+            ############################################################################################################
+            print("Step 4: get tariffs")
+            tariff_dict = dict()
+            for energy_item_id in energy_item_list:
+                tariff_dict[energy_item_id] = tariff.get_energy_item_tariffs(tenant['cost_center_id'],
+                                                                             energy_item_id,
+                                                                             start_datetime_utc,
+                                                                             end_datetime_utc)
+            ############################################################################################################
+            # Step 5: calculate billing by multiplying energy with tariff
+            ############################################################################################################
+            print("Step 5: calculate billing by multiplying energy with tariff")
+            billing_dict = dict()
+
+            if len(energy_dict) > 0:
+                for current_datetime_utc in energy_dict.keys():
+                    billing_dict[current_datetime_utc] = dict()
+                    for energy_item_id in energy_item_list:
+                        current_tariff = tariff_dict[energy_item_id].get(current_datetime_utc)
+                        current_energy = energy_dict[current_datetime_utc].get(energy_item_id)
+                        if current_tariff is not None \
+                                and isinstance(current_tariff, Decimal) \
+                                and current_energy is not None \
+                                and isinstance(current_energy, Decimal):
+                            billing_dict[current_datetime_utc][energy_item_id] = \
+                                current_energy * current_tariff
+
+                    if len(billing_dict[current_datetime_utc]) == 0:
+                        del billing_dict[current_datetime_utc]
+
+            ############################################################################################################
+            # Step 6: save billing data to billing database
+            ############################################################################################################
+            print("Step 6: save billing data to billing database")
+
+            if len(billing_dict) > 0:
+                try:
+                    add_values = (" INSERT INTO tbl_tenant_input_item_hourly "
+                                  "             (tenant_id, "
+                                  "              energy_item_id, "
+                                  "              start_datetime_utc, "
+                                  "              actual_value) "
+                                  " VALUES  ")
+
+                    for current_datetime_utc in billing_dict:
+                        for energy_item_id in energy_item_list:
+                            current_billing = billing_dict[current_datetime_utc].get(energy_item_id)
+                            if current_billing is not None and isinstance(current_billing, Decimal):
+                                add_values += " (" + str(tenant['id']) + ","
+                                add_values += " " + str(energy_item_id) + ","
+                                add_values += "'" + current_datetime_utc.isoformat()[0:19] + "',"
+                                add_values += str(billing_dict[current_datetime_utc][energy_item_id]) + "), "
+                    print("add_values:" + add_values)
+                    # trim ", " at the end of string and then execute
+                    cursor_billing_db.execute(add_values[:-2])
+                    cnx_billing_db.commit()
+                except Exception as e:
+                    logger.error("Error in step 6 of tenant_billing_input_item " + str(e))
+                    # break the for tenant loop
+                    break
+
+        # end of for tenant loop
+        if cnx_system_db:
+            cnx_system_db.close()
+        if cursor_system_db:
+            cursor_system_db.close()
+
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        if cursor_energy_db:
+            cursor_energy_db.close()
+
+        if cnx_billing_db:
+            cnx_billing_db.close()
+        if cursor_billing_db:
+            cursor_billing_db.close()
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of the outermost while loop

myems-aggregation/tenant_energy_input_category.py

@@ -0,0 +1,521 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+from multiprocessing import Pool
+import random
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all tenants
+# Step 2: Create multiprocessing pool to call worker in parallel
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all tenants
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of tenant_energy_input_category.main " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name "
+                                     " FROM tbl_tenants "
+                                     " ORDER BY id ")
+            rows_tenants = cursor_system_db.fetchall()
+
+            if rows_tenants is None or len(rows_tenants) == 0:
+                print("There isn't any tenants ")
+                # sleep and continue the outer loop to reconnect the database
+                time.sleep(60)
+                continue
+
+            tenant_list = list()
+            for row in rows_tenants:
+                tenant_list.append({"id": row[0], "name": row[1]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of tenant_energy_input_category.main " + str(e))
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        finally:
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+
+        print("Got all tenants in MyEMS System Database")
+
+        # shuffle the tenant list for randomly calculating the meter hourly value
+        random.shuffle(tenant_list)
+
+        ################################################################################################################
+        # Step 2: Create multiprocessing pool to call worker in parallel
+        ################################################################################################################
+        p = Pool(processes=config.pool_size)
+        error_list = p.map(worker, tenant_list)
+        p.close()
+        p.join()
+
+        for error in error_list:
+            if error is not None and len(error) > 0:
+                logger.error(error)
+
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of outer while
+
+
+########################################################################################################################
+# PROCEDURES:
+#   Step 1: get all input meters associated with the tenant
+#   Step 2: get all input virtual meters associated with the tenant
+#   Step 3: get all input offline meters associated with the tenant
+#   Step 4: determine start datetime and end datetime to aggregate
+#   Step 5: for each meter in list, get energy input data from energy database
+#   Step 6: for each virtual meter in list, get energy input data from energy database
+#   Step 7: for each offline meter in list, get energy input data from energy database
+#   Step 8: determine common time slot to aggregate
+#   Step 9: aggregate energy data in the common time slot by energy categories and hourly
+#   Step 10: save energy data to energy database
+#
+# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter
+########################################################################################################################
+
+def worker(tenant):
+    ####################################################################################################################
+    # Step 1: get all input meters associated with the tenant
+    ####################################################################################################################
+    print("Step 1: get all input meters associated with the tenant " + str(tenant['name']))
+
+    meter_list = list()
+    cnx_system_db = None
+    cursor_system_db = None
+    try:
+        cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+        cursor_system_db = cnx_system_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 1.1 of tenant_energy_input_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_meters m, tbl_tenants_meters tm "
+                                 " WHERE m.id = tm.meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND tm.tenant_id = %s ",
+                                 (tenant['id'],))
+        rows_meters = cursor_system_db.fetchall()
+
+        if rows_meters is not None and len(rows_meters) > 0:
+            for row in rows_meters:
+                meter_list.append({"id": row[0],
+                                   "name": row[1],
+                                   "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 1.2 of tenant_energy_input_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 2: get all input virtual meters associated with the tenant
+    ####################################################################################################################
+    print("Step 2: get all input virtual meters associated with the tenant")
+    virtual_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_virtual_meters m, tbl_tenants_virtual_meters tm "
+                                 " WHERE m.id = tm.virtual_meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND tm.tenant_id = %s ",
+                                 (tenant['id'],))
+        rows_virtual_meters = cursor_system_db.fetchall()
+
+        if rows_virtual_meters is not None and len(rows_virtual_meters) > 0:
+            for row in rows_virtual_meters:
+                virtual_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 2.1 of tenant_energy_input_category.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 3: get all input offline meters associated with the tenant
+    ####################################################################################################################
+    print("Step 3: get all input offline meters associated with the tenant")
+
+    offline_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_category_id "
+                                 " FROM tbl_offline_meters m, tbl_tenants_offline_meters tm "
+                                 " WHERE m.id = tm.offline_meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND tm.tenant_id = %s ",
+                                 (tenant['id'],))
+        rows_offline_meters = cursor_system_db.fetchall()
+
+        if rows_offline_meters is not None and len(rows_offline_meters) > 0:
+            for row in rows_offline_meters:
+                offline_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_category_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 3.1 of tenant_energy_input_category.worker " + str(e)
+        print(error_string)
+        return error_string
+    finally:
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+
+    ####################################################################################################################
+    # stop to the next tenant if this tenant is empty
+    ####################################################################################################################
+    if (meter_list is None or len(meter_list) == 0) and \
+            (virtual_meter_list is None or len(virtual_meter_list) == 0) and \
+            (offline_meter_list is None or len(offline_meter_list) == 0):
+        print("This is an empty tenant ")
+        return None
+
+    ####################################################################################################################
+    # Step 4: determine start datetime and end datetime to aggregate
+    ####################################################################################################################
+    print("Step 4: determine start datetime and end datetime to aggregate")
+    cnx_energy_db = None
+    cursor_energy_db = None
+    try:
+        cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+        cursor_energy_db = cnx_energy_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 4.1 of tenant_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        query = (" SELECT MAX(start_datetime_utc) "
+                 " FROM tbl_tenant_input_category_hourly "
+                 " WHERE tenant_id = %s ")
+        cursor_energy_db.execute(query, (tenant['id'],))
+        row_datetime = cursor_energy_db.fetchone()
+        start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+        start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+        if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+            # replace second and microsecond with 0
+            # note: do not replace minute in case of calculating in half hourly
+            start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+            # start from the next time slot
+            start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+        end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None)
+
+        print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]
+              + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19])
+
+    except Exception as e:
+        error_string = "Error in step 4.2 of tenant_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 5: for each meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_meter_hourly = dict()
+    try:
+        if meter_list is not None and len(meter_list) > 0:
+            for meter in meter_list:
+                meter_id = str(meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_meter_hourly "
+                         " WHERE meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_meter_hourly[meter_id] = None
+                else:
+                    energy_meter_hourly[meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1]
+    except Exception as e:
+        error_string = "Error in step 5.1 of tenant_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 6: for each virtual meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_virtual_meter_hourly = dict()
+    if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+        try:
+            for virtual_meter in virtual_meter_list:
+                virtual_meter_id = str(virtual_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_virtual_meter_hourly "
+                         " WHERE virtual_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_virtual_meter_hourly[virtual_meter_id] = None
+                else:
+                    energy_virtual_meter_hourly[virtual_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1]
+        except Exception as e:
+            error_string = "Error in step 6.1 of tenant_energy_input_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 7: for each offline meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_offline_meter_hourly = dict()
+    if offline_meter_list is not None and len(offline_meter_list) > 0:
+        try:
+            for offline_meter in offline_meter_list:
+                offline_meter_id = str(offline_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_offline_meter_hourly "
+                         " WHERE offline_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_offline_meter_hourly[offline_meter_id] = None
+                else:
+                    energy_offline_meter_hourly[offline_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1]
+
+        except Exception as e:
+            error_string = "Error in step 7.1 of tenant_energy_input_category.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 8: determine common time slot to aggregate
+    ####################################################################################################################
+
+    common_start_datetime_utc = start_datetime_utc
+    common_end_datetime_utc = end_datetime_utc
+
+    print("Getting common time slot of energy values for all meters")
+    if energy_meter_hourly is not None and len(energy_meter_hourly) > 0:
+        for meter_id, energy_hourly in energy_meter_hourly.items():
+            if energy_hourly is None or len(energy_hourly) == 0:
+                common_start_datetime_utc = None
+                common_end_datetime_utc = None
+                break
+            else:
+                if common_start_datetime_utc < min(energy_hourly.keys()):
+                    common_start_datetime_utc = min(energy_hourly.keys())
+                if common_end_datetime_utc > max(energy_hourly.keys()):
+                    common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all virtual meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_virtual_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all offline meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_offline_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \
+            (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \
+            (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0):
+        # There isn't any energy data
+        print("There isn't any energy data")
+        # continue the for tenant loop to the next tenant
+        print("continue the for tenant loop to the next tenant")
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        return None
+
+    print("common_start_datetime_utc: " + str(common_start_datetime_utc))
+    print("common_end_datetime_utc: " + str(common_end_datetime_utc))
+
+    ####################################################################################################################
+    # Step 9: aggregate energy data in the common time slot by energy categories and hourly
+    ####################################################################################################################
+
+    print("Step 9: aggregate energy data in the common time slot by energy categories and hourly")
+    aggregated_values = list()
+    try:
+        current_datetime_utc = common_start_datetime_utc
+        while common_start_datetime_utc is not None \
+                and common_end_datetime_utc is not None \
+                and current_datetime_utc <= common_end_datetime_utc:
+            aggregated_value = dict()
+            aggregated_value['start_datetime_utc'] = current_datetime_utc
+            aggregated_value['meta_data'] = dict()
+
+            if meter_list is not None and len(meter_list) > 0:
+                for meter in meter_list:
+                    meter_id = str(meter['id'])
+                    energy_category_id = meter['energy_category_id']
+                    actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+                for virtual_meter in virtual_meter_list:
+                    virtual_meter_id = str(virtual_meter['id'])
+                    energy_category_id = virtual_meter['energy_category_id']
+                    actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            if offline_meter_list is not None and len(offline_meter_list) > 0:
+                for offline_meter in offline_meter_list:
+                    offline_meter_id = str(offline_meter['id'])
+                    energy_category_id = offline_meter['energy_category_id']
+                    actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_category_id] = \
+                        aggregated_value['meta_data'].get(energy_category_id, Decimal(0.0)) + actual_value
+
+            aggregated_values.append(aggregated_value)
+
+            current_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+    except Exception as e:
+        error_string = "Error in step 9 of tenant_energy_input_category.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 10: save energy data to energy database
+    ####################################################################################################################
+    print("Step 10: save energy data to energy database")
+
+    if len(aggregated_values) > 0:
+        try:
+            add_values = (" INSERT INTO tbl_tenant_input_category_hourly "
+                          "             (tenant_id, "
+                          "              energy_category_id, "
+                          "              start_datetime_utc, "
+                          "              actual_value) "
+                          " VALUES  ")
+
+            for aggregated_value in aggregated_values:
+                for energy_category_id, actual_value in aggregated_value['meta_data'].items():
+                    add_values += " (" + str(tenant['id']) + ","
+                    add_values += " " + str(energy_category_id) + ","
+                    add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "',"
+                    add_values += str(actual_value) + "), "
+            print("add_values:" + add_values)
+            # trim ", " at the end of string and then execute
+            cursor_energy_db.execute(add_values[:-2])
+            cnx_energy_db.commit()
+
+        except Exception as e:
+            error_string = "Error in step 10.1 of tenant_energy_input_category.worker " + str(e)
+            print(error_string)
+            return error_string
+        finally:
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+    else:
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()

myems-aggregation/tenant_energy_input_item.py

@@ -0,0 +1,524 @@
+import time
+from datetime import datetime, timedelta
+from decimal import Decimal
+import mysql.connector
+from multiprocessing import Pool
+import random
+import config
+
+
+########################################################################################################################
+# PROCEDURES
+# Step 1: get all tenants
+# Step 2: Create multiprocessing pool to call worker in parallel
+########################################################################################################################
+
+
+def main(logger):
+
+    while True:
+        # the outermost while loop
+        ################################################################################################################
+        # Step 1: get all tenants
+        ################################################################################################################
+        cnx_system_db = None
+        cursor_system_db = None
+        try:
+            cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+            cursor_system_db = cnx_system_db.cursor()
+        except Exception as e:
+            logger.error("Error in step 1.1 of tenant_energy_input_item.main " + str(e))
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        print("Connected to MyEMS System Database")
+
+        try:
+            cursor_system_db.execute(" SELECT id, name "
+                                     " FROM tbl_tenants "
+                                     " ORDER BY id ")
+            rows_tenants = cursor_system_db.fetchall()
+
+            if rows_tenants is None or len(rows_tenants) == 0:
+                print("There isn't any tenants ")
+                # sleep and continue the outer loop to reconnect the database
+                time.sleep(60)
+                continue
+
+            tenant_list = list()
+            for row in rows_tenants:
+                tenant_list.append({"id": row[0], "name": row[1]})
+
+        except Exception as e:
+            logger.error("Error in step 1.2 of tenant_energy_input_item.main " + str(e))
+            # sleep and continue the outer loop to reconnect the database
+            time.sleep(60)
+            continue
+        finally:
+            if cursor_system_db:
+                cursor_system_db.close()
+            if cnx_system_db:
+                cnx_system_db.close()
+
+        print("Got all tenants in MyEMS System Database")
+
+        # shuffle the tenant list for randomly calculating the meter hourly value
+        random.shuffle(tenant_list)
+
+        ################################################################################################################
+        # Step 2: Create multiprocessing pool to call worker in parallel
+        ################################################################################################################
+        p = Pool(processes=config.pool_size)
+        error_list = p.map(worker, tenant_list)
+        p.close()
+        p.join()
+
+        for error in error_list:
+            if error is not None and len(error) > 0:
+                logger.error(error)
+
+        print("go to sleep 300 seconds...")
+        time.sleep(300)
+        print("wake from sleep, and continue to work...")
+    # end of outer while
+
+
+########################################################################################################################
+# PROCEDURES:
+#   Step 1: get all input meters associated with the tenant
+#   Step 2: get all input virtual meters associated with the tenant
+#   Step 3: get all input offline meters associated with the tenant
+#   Step 4: determine start datetime and end datetime to aggregate
+#   Step 5: for each meter in list, get energy input data from energy database
+#   Step 6: for each virtual meter in list, get energy input data from energy database
+#   Step 7: for each offline meter in list, get energy input data from energy database
+#   Step 8: determine common time slot to aggregate
+#   Step 9: aggregate energy data in the common time slot by energy items and hourly
+#   Step 10: save energy data to energy database
+#
+# NOTE: returns None or the error string because that the logger object cannot be passed in as parameter
+########################################################################################################################
+
+def worker(tenant):
+    ####################################################################################################################
+    # Step 1: get all input meters associated with the tenant
+    ####################################################################################################################
+    print("Step 1: get all input meters associated with the tenant " + str(tenant['name']))
+
+    meter_list = list()
+    cnx_system_db = None
+    cursor_system_db = None
+    try:
+        cnx_system_db = mysql.connector.connect(**config.myems_system_db)
+        cursor_system_db = cnx_system_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 1.1 of tenant_energy_input_item.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id "
+                                 " FROM tbl_meters m, tbl_tenants_meters tm "
+                                 " WHERE m.id = tm.meter_id "
+                                 "       AND m.is_counted = true "
+                                 "       AND m.energy_item_id is NOT NULL "
+                                 "       AND tm.tenant_id = %s ",
+                                 (tenant['id'],))
+        rows_meters = cursor_system_db.fetchall()
+
+        if rows_meters is not None and len(rows_meters) > 0:
+            for row in rows_meters:
+                meter_list.append({"id": row[0],
+                                   "name": row[1],
+                                   "energy_item_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 1.2 of tenant_energy_input_item.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 2: get all input virtual meters associated with the tenant
+    ####################################################################################################################
+    print("Step 2: get all input virtual meters associated with the tenant")
+    virtual_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id "
+                                 " FROM tbl_virtual_meters m, tbl_tenants_virtual_meters tm "
+                                 " WHERE m.id = tm.virtual_meter_id "
+                                 "       AND m.energy_item_id is NOT NULL "
+                                 "       AND m.is_counted = true "
+                                 "       AND tm.tenant_id = %s ",
+                                 (tenant['id'],))
+        rows_virtual_meters = cursor_system_db.fetchall()
+
+        if rows_virtual_meters is not None and len(rows_virtual_meters) > 0:
+            for row in rows_virtual_meters:
+                virtual_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_item_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 2.1 of tenant_energy_input_item.worker " + str(e)
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 3: get all input offline meters associated with the tenant
+    ####################################################################################################################
+    print("Step 3: get all input offline meters associated with the tenant")
+
+    offline_meter_list = list()
+
+    try:
+        cursor_system_db.execute(" SELECT m.id, m.name, m.energy_item_id "
+                                 " FROM tbl_offline_meters m, tbl_tenants_offline_meters tm "
+                                 " WHERE m.id = tm.offline_meter_id "
+                                 "       AND m.energy_item_id is NOT NULL "
+                                 "       AND m.is_counted = true "
+                                 "       AND tm.tenant_id = %s ",
+                                 (tenant['id'],))
+        rows_offline_meters = cursor_system_db.fetchall()
+
+        if rows_offline_meters is not None and len(rows_offline_meters) > 0:
+            for row in rows_offline_meters:
+                offline_meter_list.append({"id": row[0],
+                                           "name": row[1],
+                                           "energy_item_id": row[2]})
+
+    except Exception as e:
+        error_string = "Error in step 3.1 of tenant_energy_input_item.worker " + str(e)
+        print(error_string)
+        return error_string
+    finally:
+        if cursor_system_db:
+            cursor_system_db.close()
+        if cnx_system_db:
+            cnx_system_db.close()
+
+    ####################################################################################################################
+    # stop to the next tenant if this tenant is empty
+    ####################################################################################################################
+    if (meter_list is None or len(meter_list) == 0) and \
+            (virtual_meter_list is None or len(virtual_meter_list) == 0) and \
+            (offline_meter_list is None or len(offline_meter_list) == 0):
+        print("This is an empty tenant ")
+        return None
+
+    ####################################################################################################################
+    # Step 4: determine start datetime and end datetime to aggregate
+    ####################################################################################################################
+    print("Step 4: determine start datetime and end datetime to aggregate")
+    cnx_energy_db = None
+    cursor_energy_db = None
+    try:
+        cnx_energy_db = mysql.connector.connect(**config.myems_energy_db)
+        cursor_energy_db = cnx_energy_db.cursor()
+    except Exception as e:
+        error_string = "Error in step 4.1 of tenant_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    try:
+        query = (" SELECT MAX(start_datetime_utc) "
+                 " FROM tbl_tenant_input_item_hourly "
+                 " WHERE tenant_id = %s ")
+        cursor_energy_db.execute(query, (tenant['id'],))
+        row_datetime = cursor_energy_db.fetchone()
+        start_datetime_utc = datetime.strptime(config.start_datetime_utc, '%Y-%m-%d %H:%M:%S')
+        start_datetime_utc = start_datetime_utc.replace(minute=0, second=0, microsecond=0, tzinfo=None)
+
+        if row_datetime is not None and len(row_datetime) > 0 and isinstance(row_datetime[0], datetime):
+            # replace second and microsecond with 0
+            # note: do not replace minute in case of calculating in half hourly
+            start_datetime_utc = row_datetime[0].replace(second=0, microsecond=0, tzinfo=None)
+            # start from the next time slot
+            start_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+        end_datetime_utc = datetime.utcnow().replace(second=0, microsecond=0, tzinfo=None)
+
+        print("start_datetime_utc: " + start_datetime_utc.isoformat()[0:19]
+              + "end_datetime_utc: " + end_datetime_utc.isoformat()[0:19])
+
+    except Exception as e:
+        error_string = "Error in step 4.2 of tenant_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 5: for each meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_meter_hourly = dict()
+    try:
+        if meter_list is not None and len(meter_list) > 0:
+            for meter in meter_list:
+                meter_id = str(meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_meter_hourly "
+                         " WHERE meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_meter_hourly[meter_id] = None
+                else:
+                    energy_meter_hourly[meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_meter_hourly[meter_id][row_energy_value[0]] = row_energy_value[1]
+    except Exception as e:
+        error_string = "Error in step 5.1 of tenant_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 6: for each virtual meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_virtual_meter_hourly = dict()
+    if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+        try:
+            for virtual_meter in virtual_meter_list:
+                virtual_meter_id = str(virtual_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_virtual_meter_hourly "
+                         " WHERE virtual_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (virtual_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_virtual_meter_hourly[virtual_meter_id] = None
+                else:
+                    energy_virtual_meter_hourly[virtual_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_virtual_meter_hourly[virtual_meter_id][row_energy_value[0]] = row_energy_value[1]
+        except Exception as e:
+            error_string = "Error in step 6.1 of tenant_energy_input_item.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 7: for each offline meter in list, get energy input data from energy database
+    ####################################################################################################################
+    energy_offline_meter_hourly = dict()
+    if offline_meter_list is not None and len(offline_meter_list) > 0:
+        try:
+            for offline_meter in offline_meter_list:
+                offline_meter_id = str(offline_meter['id'])
+
+                query = (" SELECT start_datetime_utc, actual_value "
+                         " FROM tbl_offline_meter_hourly "
+                         " WHERE offline_meter_id = %s "
+                         "       AND start_datetime_utc >= %s "
+                         "       AND start_datetime_utc < %s "
+                         " ORDER BY start_datetime_utc ")
+                cursor_energy_db.execute(query, (offline_meter_id, start_datetime_utc, end_datetime_utc,))
+                rows_energy_values = cursor_energy_db.fetchall()
+                if rows_energy_values is None or len(rows_energy_values) == 0:
+                    energy_offline_meter_hourly[offline_meter_id] = None
+                else:
+                    energy_offline_meter_hourly[offline_meter_id] = dict()
+                    for row_energy_value in rows_energy_values:
+                        energy_offline_meter_hourly[offline_meter_id][row_energy_value[0]] = row_energy_value[1]
+
+        except Exception as e:
+            error_string = "Error in step 7.1 of tenant_energy_input_item.worker " + str(e)
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+            print(error_string)
+            return error_string
+
+    ####################################################################################################################
+    # Step 8: determine common time slot to aggregate
+    ####################################################################################################################
+
+    common_start_datetime_utc = start_datetime_utc
+    common_end_datetime_utc = end_datetime_utc
+
+    print("Getting common time slot of energy values for all meters")
+    if energy_meter_hourly is not None and len(energy_meter_hourly) > 0:
+        for meter_id, energy_hourly in energy_meter_hourly.items():
+            if energy_hourly is None or len(energy_hourly) == 0:
+                common_start_datetime_utc = None
+                common_end_datetime_utc = None
+                break
+            else:
+                if common_start_datetime_utc < min(energy_hourly.keys()):
+                    common_start_datetime_utc = min(energy_hourly.keys())
+                if common_end_datetime_utc > max(energy_hourly.keys()):
+                    common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all virtual meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_virtual_meter_hourly is not None and len(energy_virtual_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_virtual_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    print("Getting common time slot of energy values for all offline meters")
+    if common_start_datetime_utc is not None and common_start_datetime_utc is not None:
+        if energy_offline_meter_hourly is not None and len(energy_offline_meter_hourly) > 0:
+            for meter_id, energy_hourly in energy_offline_meter_hourly.items():
+                if energy_hourly is None or len(energy_hourly) == 0:
+                    common_start_datetime_utc = None
+                    common_end_datetime_utc = None
+                    break
+                else:
+                    if common_start_datetime_utc < min(energy_hourly.keys()):
+                        common_start_datetime_utc = min(energy_hourly.keys())
+                    if common_end_datetime_utc > max(energy_hourly.keys()):
+                        common_end_datetime_utc = max(energy_hourly.keys())
+
+    if (energy_meter_hourly is None or len(energy_meter_hourly) == 0) and \
+            (energy_virtual_meter_hourly is None or len(energy_virtual_meter_hourly) == 0) and \
+            (energy_offline_meter_hourly is None or len(energy_offline_meter_hourly) == 0):
+        # There isn't any energy data
+        print("There isn't any energy data")
+        # continue the for tenant loop to the next tenant
+        print("continue the for tenant loop to the next tenant")
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        return None
+
+    print("common_start_datetime_utc: " + str(common_start_datetime_utc))
+    print("common_end_datetime_utc: " + str(common_end_datetime_utc))
+
+    ####################################################################################################################
+    # Step 9: aggregate energy data in the common time slot by energy items and hourly
+    ####################################################################################################################
+
+    print("Step 9: aggregate energy data in the common time slot by energy items and hourly")
+    aggregated_values = list()
+    try:
+        current_datetime_utc = common_start_datetime_utc
+        while common_start_datetime_utc is not None \
+                and common_end_datetime_utc is not None \
+                and current_datetime_utc <= common_end_datetime_utc:
+            aggregated_value = dict()
+            aggregated_value['start_datetime_utc'] = current_datetime_utc
+            aggregated_value['meta_data'] = dict()
+
+            if meter_list is not None and len(meter_list) > 0:
+                for meter in meter_list:
+                    meter_id = str(meter['id'])
+                    energy_item_id = meter['energy_item_id']
+                    actual_value = energy_meter_hourly[meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_item_id] = \
+                        aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value
+
+            if virtual_meter_list is not None and len(virtual_meter_list) > 0:
+                for virtual_meter in virtual_meter_list:
+                    virtual_meter_id = str(virtual_meter['id'])
+                    energy_item_id = virtual_meter['energy_item_id']
+                    actual_value = energy_virtual_meter_hourly[virtual_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_item_id] = \
+                        aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value
+
+            if offline_meter_list is not None and len(offline_meter_list) > 0:
+                for offline_meter in offline_meter_list:
+                    offline_meter_id = str(offline_meter['id'])
+                    energy_item_id = offline_meter['energy_item_id']
+                    actual_value = energy_offline_meter_hourly[offline_meter_id].get(current_datetime_utc, Decimal(0.0))
+                    aggregated_value['meta_data'][energy_item_id] = \
+                        aggregated_value['meta_data'].get(energy_item_id, Decimal(0.0)) + actual_value
+
+            aggregated_values.append(aggregated_value)
+
+            current_datetime_utc += timedelta(minutes=config.minutes_to_count)
+
+    except Exception as e:
+        error_string = "Error in step 9 of tenant_energy_input_item.worker " + str(e)
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()
+        print(error_string)
+        return error_string
+
+    ####################################################################################################################
+    # Step 10: save energy data to energy database
+    ####################################################################################################################
+    print("Step 10: save energy data to energy database")
+
+    if len(aggregated_values) > 0:
+        try:
+            add_values = (" INSERT INTO tbl_tenant_input_item_hourly "
+                          "             (tenant_id, "
+                          "              energy_item_id, "
+                          "              start_datetime_utc, "
+                          "              actual_value) "
+                          " VALUES  ")
+
+            for aggregated_value in aggregated_values:
+                for energy_item_id, actual_value in aggregated_value['meta_data'].items():
+                    add_values += " (" + str(tenant['id']) + ","
+                    add_values += " " + str(energy_item_id) + ","
+                    add_values += "'" + aggregated_value['start_datetime_utc'].isoformat()[0:19] + "',"
+                    add_values += str(actual_value) + "), "
+            print("add_values:" + add_values)
+            # trim ", " at the end of string and then execute
+            cursor_energy_db.execute(add_values[:-2])
+            cnx_energy_db.commit()
+
+        except Exception as e:
+            error_string = "Error in step 10.1 of tenant_energy_input_item.worker " + str(e)
+            print(error_string)
+            return error_string
+        finally:
+            if cursor_energy_db:
+                cursor_energy_db.close()
+            if cnx_energy_db:
+                cnx_energy_db.close()
+    else:
+        if cursor_energy_db:
+            cursor_energy_db.close()
+        if cnx_energy_db:
+            cnx_energy_db.close()

myems-aggregation/test_tariff.py

@@ -0,0 +1,33 @@
+from datetime import datetime
+import tariff
+
+
+def main():
+    """main"""
+    print('Testing get_energy_category_tariffs ...')
+    cost_center_id = 1
+    energy_category_id = 1
+    start_date_time_utc = datetime.strptime('2020-06-29 16:00:00', '%Y-%m-%d %H:%M:%S')
+    end_date_time_utc = datetime.strptime('2020-07-01 15:59:59', '%Y-%m-%d %H:%M:%S')
+    tariffs = tariff.get_energy_category_tariffs(cost_center_id,
+                                                 energy_category_id,
+                                                 start_date_time_utc,
+                                                 end_date_time_utc)
+    for k, v in sorted(tariffs.items()):
+        print(k, v)
+
+    print('Testing get_energy_item_tariffs ...')
+    cost_center_id = 1
+    energy_item_id = 1
+    start_date_time_utc = datetime.strptime('2020-06-29 16:00:00', '%Y-%m-%d %H:%M:%S')
+    end_date_time_utc = datetime.strptime('2020-07-01 15:59:59', '%Y-%m-%d %H:%M:%S')
+    tariffs = tariff.get_energy_item_tariffs(cost_center_id,
+                                             energy_item_id,
+                                             start_date_time_utc,
+                                             end_date_time_utc)
+    for k, v in sorted(tariffs.items()):
+        print(k, v)
+
+
+if __name__ == "__main__":
+    main()