Rewrite electricity-logger to use an sqlite3 database

include/electricity/logger/database.hpp

@@ -0,0 +1,15 @@
+#pragma once
+#include <electricity/logger/dsmr.hpp>
+#include <sqlite3.h>
+#include <string>
+
+class Database {
+private:
+    sqlite3 * connectionPtr;
+
+public:
+    void Insert(DSMR::Data const & data, time_t const time);
+
+    Database(std::string const & databaseFilePath);
+    ~Database();
+};

include/electricity/logger/dsmr.hpp

@@ -0,0 +1,73 @@
+#pragma once
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+//#include <iostream> // debug
+
+namespace DSMR
+{
+    enum class LineTag
+    {
+        Unknown = -1,
+        DSMRversion,
+        DateTimeStamp,
+        SerialNo,
+        TotalPowerConsumedTariff1,
+        TotalPowerConsumedTariff2,
+        TotalReturnedPowerTariff1,
+        TotalReturnedPowerTariff2,
+        CurrentTarif,
+        CurrentPowerConsumption,
+        CurrentPowerReturn,
+        PowerFailureCount,
+        PowerFailureLongCount,
+        PowerFailureEventLog,
+        VoltageL1SagCount,
+        VoltageL2SagCount,
+        VoltageL3SagCount,
+        VoltageL1SwellCount,
+        VoltageL2SwellCount,
+        VoltageL3SwellCount,
+        TextMessageMaxChar,
+        InstantL1VoltageResolution,
+        InstantL2VoltageResolution,
+        InstantL3VoltageResolution,
+        InstantL1CurrentResolution,
+        InstantL2CurrentResolution,
+        InstantL3CurrentResolution,
+        InstantL1ActivePowerResolution,
+        InstantL2ActivePowerResolution,
+        InstantL3ActivePowerResolution,
+        InstantL1ActivePowerResolutionA,
+        InstantL2ActivePowerResolutionA,
+        InstantL3ActivePowerResolutionA,
+        DeviceType,
+        GasDeviceIdentifier,
+        GasTotalConsumptionLog
+    };
+
+    // DSMR output parser for Landis Gyr E350, we only extract things that interest us.
+    class Data {
+    private:
+        static const std::unordered_map<std::string, LineTag> & GetMap();
+
+        static void RemoveUnit(std::string & value);
+
+        // Single argument lines only
+        static std::pair<std::string, std::string> GetKeyValuePair(const std::string & line);
+
+    public:
+        double currentPowerUsageKw = 0.0, currentPowerReturnKw = 0.0;
+        double totalPowerConsumptionDayKwh = 0.0, totalPowerConsumptionNightKwh = 0.0;
+        double totalPowerReturnedDayKwh = 0.0, totalPowerReturnedNightKwh = 0.0;
+        bool usingDayTarif = true;
+        std::string gasTimestamp = "";
+        double gasConsumptionCubicMeters = 0.0; // m^3
+
+        void ParseLine(const std::string & line);
+        void ParseLines(const std::vector<std::string> & lines);
+
+        std::string GetFormattedString(char const separator) const;
+    };
+}

include/electricity/logger/serialport.hpp

@@ -0,0 +1,24 @@
+#pragma once
+#include <string>
+#include <termios.h>
+
+class SerialPort {
+private:
+    const int device;
+    termios configuration, oldConfiguration;
+
+    void SetAttributes(const speed_t baudrate = B115200);
+
+protected:
+public:
+    std::string ReadLine();
+
+    SerialPort(const int fd);
+
+    ~SerialPort();
+
+    SerialPort(const SerialPort &) = delete;
+    SerialPort(SerialPort &&) = delete;
+    SerialPort & operator=(const SerialPort &) = delete;
+    SerialPort & operator=(SerialPort &&) = delete;
+};

include/electricity/server/api.hpp

@@ -0,0 +1,10 @@
+#pragma once
+#include <pistache/http.h>
+#include <pistache/router.h>
+
+namespace Server::Api
+{
+    void GetDay(Pistache::Http::Request const & request, Pistache::Http::ResponseWriter responseWrite);
+
+    void SetupRouting(Pistache::Rest::Router & router);
+}

include/electricity/server/configuration.hpp

@@ -0,0 +1,33 @@
+#pragma once
+#include <chrono>
+#include <mutex>
+#include <string>
+
+namespace Server
+{
+    class Configuration {
+    private:
+        std::string logDirectory;
+        std::string serverDomain;
+        std::string lastExternalIp;
+        std::chrono::time_point<std::chrono::steady_clock> lastIpCheckTimePoint;
+        std::mutex externalIpRefreshMutex;
+
+        Configuration();
+        Configuration(Configuration & other) = delete;
+        Configuration(Configuration && other) = delete;
+        Configuration & operator=(Configuration & other) = delete;
+        Configuration & operator=(Configuration && other) = delete;
+
+        void RefreshExternalIp();
+        bool ExternalIpRequiresRefresh() const;
+
+    public:
+        void Setup(std::string & electricityLogDirectory, std::string const & serverDomain);
+
+        std::string const & GetLogDirectory() const;
+        std::string const & GetExternalServerIp();
+
+        static Configuration & Get();
+    };
+}

include/electricity/server/database.hpp

@@ -0,0 +1,8 @@
+#include <string>
+#include <util/date.hpp>
+
+namespace Server::Database
+{
+    std::string GetDetailedJsonOf(Util::Date const & date);
+    std::string GetSummaryJsonOf(Util::Date const & startDate, Util::Date const & stopDate);
+}

include/migrator/migrations.hpp

@@ -0,0 +1,9 @@
+#pragma once
+#include <string>
+
+namespace Migrations
+{
+    int EpochToDateTime(std::string const & sourceDatabase, std::string const & destinationDatabase);
+
+    int UpdateSummaryTable(std::string const & sourceDatabase, std::string const & destinationDatabase);
+}

include/migrator/transaction.hpp

@@ -0,0 +1,24 @@
+#pragma once
+#include <sqlite3.h>
+#include <sstream>
+
+class Transaction {
+private:
+    sqlite3 * const destination;
+    unsigned queryCount;
+    std::stringstream queryStream;
+
+    void Reset();
+
+public:
+    void AddStatement(std::string const & statement);
+
+    unsigned StatementCount() const;
+
+    // Runs the statements inserted as a single transaction
+    // Returns a SQLite status code (0 = OK) and then resets
+    // to a new transaction
+    int Execute();
+
+    Transaction(sqlite3 * const databaseToInsertIn);
+};

include/solar/logger/database.hpp

@@ -0,0 +1,25 @@
+#pragma once
+#include <cstdint>
+#include <sqlite3.h>
+#include <string>
+
+struct Row
+{
+    time_t epochTime;
+    std::int32_t watt;
+    double kilowattPerHour;
+};
+
+class Database {
+private:
+    sqlite3 * connectionPtr;
+
+    std::string ToSqlInsertStatement(Row const & row) const;
+    std::string ToSqlUpsertStatement(Row const & row) const;
+
+public:
+    bool Insert(Row & row);
+
+    Database(std::string const & databasePath);
+    ~Database();
+};

include/solar/logger/zeverdata.hpp

@@ -0,0 +1,33 @@
+#pragma once
+#include <cstdio>
+#include <ctime>
+#include <curl/curl.h> // tested with libcurl4-openSSH
+#include <sstream>
+#include <string>
+
+/*
+	This class holds all the data retrieved from the Zeverlution combox /
+	Zeverlution Sxxxx smart inverters / ZeverSolar box.
+
+	For output formatting it uses a timestamp to log the datetime alongside the
+	watts and kilowatts.
+*/
+
+struct ZeverData
+{
+    int number0, number1;
+    std::string registeryID, registeryKey, hardwareVersion, appVersion, wifiVersion;
+    std::string timeValue, dateValue, zeverCloudStatus;
+    int number3;
+    std::string inverterSN;
+    int watt;
+    double kilowattPerHour;
+    std::string OKmsg, ERRORmsg;
+
+    // Parses the data string coming from the zeverlution home.cgi webpage
+    bool ParseString(const std::string & str);
+
+    bool FetchDataFromURL(const std::string & url, const unsigned int timeout);
+
+    ZeverData();
+};

include/solar/server/api.hpp

@@ -0,0 +1,10 @@
+#pragma once
+#include <pistache/router.h>
+
+namespace Api
+{
+    void GetDay(Pistache::Http::Request const & request, Pistache::Http::ResponseWriter responseWrite);
+    void GetMonth(Pistache::Http::Request const & request, Pistache::Http::ResponseWriter responseWrite);
+
+    void SetupRouting(Pistache::Rest::Router & router);
+}

include/solar/server/configuration.hpp

@@ -0,0 +1,18 @@
+#pragma once
+#include <solar/server/database/database.hpp>
+#include <string>
+
+class Configuration {
+private:
+    Database::Database database;
+
+    Configuration();
+
+    friend int main(int, char **);
+    Configuration & SetupDatabase(std::string const & filePath);
+
+public:
+    Database::Connection GetDatabaseConnection() const;
+
+    static Configuration & Get();
+};

include/solar/server/database/connection.hpp

@@ -0,0 +1,30 @@
+#pragma once
+#include <ctime>
+#include <sqlite3.h>
+#include <string>
+#include <util/date.hpp>
+
+namespace Database
+{
+    class Connection {
+    private:
+        sqlite3 * const connectionPtr;
+
+    public:
+        Connection(sqlite3 * const databaseConnectionPtr);
+
+        // Date should be in format yyyy-mm-dd
+        // Returns a JSON array
+        std::string GetEntireDay(Util::Date const & date);
+
+        // Dates should be in format yyyy-mm-dd
+        // Returns a JSON array
+        // startDate and endDate are inclusive
+        std::string GetSummarizedPerDayRecords(Util::Date const & startDate, Util::Date const & endDate);
+
+        // Dates should be in format yyyy-mm-dd
+        // Returns a JSON array
+        // startDate and endDate are inclusive
+        std::string GetSummarizedPerMonthRecords(Util::Date const & startDate, Util::Date const & endDate);
+    };
+}

include/solar/server/database/database.hpp

@@ -0,0 +1,20 @@
+#pragma once
+#include <solar/server/database/connection.hpp>
+#include <sqlite3.h>
+#include <string>
+
+namespace Database
+{
+    class Database {
+    private:
+        sqlite3 * connectionPtr;
+
+    public:
+        bool Connect(std::string const & path);
+
+        Connection GetConnection() const;
+
+        Database();
+        ~Database();
+    };
+}

include/util/date.hpp

@@ -0,0 +1,151 @@
+#pragma once
+#include <cstdio>
+#include <ctime>
+#include <string>
+
+namespace Util
+{
+    inline bool IsValidYear(int const year) { return year > 1900 && year < 9999; }
+
+    inline bool IsValidMonth(int const month) { return month > 0 && month < 13; }
+
+    inline bool IsValidDay(int const day) { return day > 0 && day < 32; }
+
+    inline long GetEpoch(long year, unsigned month, unsigned day)
+    {
+        year -= month <= 2;
+        const long era = (year >= 0 ? year : year - 399) / 400;
+        const unsigned yoe = static_cast<unsigned>(year - era * 400); // [0, 399]
+        const unsigned doy = (153 * (month + (month > 2 ? -3 : 9)) + 2) / 5 + day - 1; // [0, 365]
+        const unsigned doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; // [0, 146096]
+
+        // Multiply by the day length in seconds
+        return (era * 146097 + doe - 719468) * (24l * 60l * 60l);
+    }
+
+    inline long ToEpoch(std::string const & date)
+    {
+        int year, month, day;
+        if(std::sscanf(date.c_str(), "%d-%d-%d", &year, &month, &day) != 3)
+        {
+            return -1;
+        }
+
+        return GetEpoch(year, month, day);
+    }
+
+    inline unsigned ToSeconds(unsigned hours, unsigned minutes, unsigned seconds)
+    {
+        return (hours * 60 * 60) + (minutes * 60) + seconds;
+    }
+
+    inline int ToSecondsFromTimeString(char const * string)
+    {
+        int hour, minute, second;
+        if(std::sscanf(string, "%d:%d:%d", &hour, &minute, &second) != 3)
+        {
+            return -1;
+        }
+
+        return ToSeconds(hour, minute, second);
+    }
+
+    inline std::string GetSqliteDate(time_t const epochTime)
+    {
+        auto const dateTime = *std::gmtime(&epochTime);
+
+        char dateStringBuffer[64];
+        std::sprintf(dateStringBuffer, "%i-%02i-%02i", dateTime.tm_year + 1900, dateTime.tm_mon + 1, dateTime.tm_mday);
+
+        return std::string(dateStringBuffer);
+    }
+
+    inline std::string GetSqliteUtcTime(time_t const epochTime)
+    {
+        auto const dateTime = *std::gmtime(&epochTime);
+
+        char timeStringBuffer[64];
+        std::sprintf(timeStringBuffer, "%02i:%02i:%02i", dateTime.tm_hour, dateTime.tm_min, dateTime.tm_sec);
+
+        return std::string(timeStringBuffer);
+    }
+
+    class Date {
+    private:
+        unsigned year;
+        unsigned month;
+        unsigned day;
+
+    public:
+        unsigned Year() const { return year; }
+        unsigned Month() const { return month; }
+        unsigned Day() const { return day; }
+
+        void AddMonth()
+        {
+            ++month;
+            if(month > 12)
+            {
+                month = 1;
+                ++year;
+            }
+        }
+
+        void SetDayToEndOfMonth() { day = 31; }
+
+        bool IsValid() const { return IsValidYear(year) && IsValidMonth(month) && IsValidDay(day); }
+
+        long ToEpoch() const { return GetEpoch(year, month, day); }
+
+        bool IsBefore(unsigned const otherYear, unsigned const otherMonth, unsigned const otherDay) const
+        {
+            return (year < otherYear) || (year == otherYear && month < otherMonth)
+                || (year == otherYear && month == otherMonth && day < otherDay);
+        }
+
+        bool IsBefore(Date const & other) const { return IsBefore(other.year, other.month, other.day); }
+
+        bool IsAfter(unsigned const otherYear, unsigned const otherMonth, unsigned const otherDay) const
+        {
+            return (year > otherYear) || (year == otherYear && month > otherMonth)
+                || (year == otherYear && month == otherMonth && day > otherDay);
+        }
+
+        std::string ToISOString() const
+        {
+            char buffer[11];
+            std::sprintf(buffer, "%04i-%02i-%02i", year, month, day);
+
+            return std::string(buffer);
+        }
+
+        bool TryParse(std::string const & date)
+        {
+            Date result;
+
+            if(sscanf(date.c_str(), "%d-%d-%d", &year, &month, &day) != 3)
+            {
+                result.year = 0;
+                result.month = 0;
+                result.day = 0;
+                return false;
+            }
+
+            return true;
+        }
+
+        Date() : year(0), month(0), day(0) { }
+
+        Date(std::string const & date) : year(0), month(0), day(0) { TryParse(date); }
+
+        Date(long const epoch)
+        {
+            std::tm dateTime = *std::gmtime(&epoch);
+            year = 1900 + dateTime.tm_year;
+            month = 1 + dateTime.tm_mon;
+            day = dateTime.tm_mday;
+        }
+
+        static Date UtcNow() { return Date(std::time(nullptr)); }
+    };
+}