tijmen/home-data-collection-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove migrator tool

Browse Source
This commit is contained in:
Tijmen van Nesselrooij
2022-06-27 20:06:57 +02:00
parent 5b09b06bcf
commit 005acd6e1f
8 changed files with 1 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -11,7 +11,6 @@ This repository houses various projects. All are listed below.
- Two pistache REST HTTP APIs, [documentation](docs/SERVER.md) - Two pistache REST HTTP APIs, [documentation](docs/SERVER.md)
- A solar panel log server, [documentation](docs/SOLAR_API.md) - A solar panel log server, [documentation](docs/SOLAR_API.md)
- An electricity log server, [documentation](docs/ELECTRICITY_API.md) - An electricity log server, [documentation](docs/ELECTRICITY_API.md)
- A migrator to convert and build data structures yielded by the loggers, [documentation](docs/MIGRATOR.md)
## Project Directory ## Project Directory
@@ -22,7 +21,6 @@ This repository houses various projects. All are listed below.
|- `src` => A folder with all the source files of the different projects that require compilation </br> |- `src` => A folder with all the source files of the different projects that require compilation </br>
|....|- `electricity-logger` => The Landis Gyr E350 logger source files </br> |....|- `electricity-logger` => The Landis Gyr E350 logger source files </br>
|....|- `electricity-server` => The pistache REST API sources for serving the electricity logs </br> |....|- `electricity-server` => The pistache REST API sources for serving the electricity logs </br>
|....|- `migrator ` => The migrator source files </br>
|....|- `solar-logger` => The Zeverlution S3000 logger source files </br> |....|- `solar-logger` => The Zeverlution S3000 logger source files </br>
|....|- `solar-server` => The pistache REST API sources for serving the solar panel logs </br> |....|- `solar-server` => The pistache REST API sources for serving the solar panel logs </br>
|- `systemd` => A folder with example systemd service files for the servers </br> |- `systemd` => A folder with example systemd service files for the servers </br>

9
include/migrator/migrations.hpp
View File

@@ -1,9 +0,0 @@
#pragma once
#include <string>
namespace Migrations
{
int EpochToDateTime(std::string const & sourceDatabase, std::string const & destinationDatabase);
int UpdateSummaryTable(std::string const & sourceDatabase, std::string const & destinationDatabase);
}

24
include/migrator/transaction.hpp
View File

@@ -1,24 +0,0 @@
#pragma once
#include <sqlite3.h>
#include <sstream>
class Transaction {
private:
sqlite3 * const destination;
unsigned queryCount;
std::stringstream queryStream;
void Reset();
public:
void AddStatement(std::string const & statement);
unsigned StatementCount() const;
// Runs the statements inserted as a single transaction
// Returns a SQLite status code (0 = OK) and then resets
// to a new transaction
int Execute();
Transaction(sqlite3 * const databaseToInsertIn);
};

12
makefile
View File

@@ -3,7 +3,6 @@ CFLAGS = -DSPDLOG_FMT_EXTERNAL -Wall -Wextra -std=c++20 -O2 -Iinclude/ -MMD
ELECTRICITY_LOGGER_LFLAGS = -lsqlite3 -lspdlog -lfmt ELECTRICITY_LOGGER_LFLAGS = -lsqlite3 -lspdlog -lfmt
SOLAR_LOGGER_LFLAGS = -lsqlite3 -lcurl SOLAR_LOGGER_LFLAGS = -lsqlite3 -lcurl
SERVER_LFLAGS = -lpistache -lsqlite3 -lstdc++fs -lspdlog -lfmt SERVER_LFLAGS = -lpistache -lsqlite3 -lstdc++fs -lspdlog -lfmt
MIGRATOR_LFLAGS = -lsqlite3
SOLAR_LOGGER_CPPS = $(shell find src/solar-logger/ -name *.cpp) SOLAR_LOGGER_CPPS = $(shell find src/solar-logger/ -name *.cpp)
SOLAR_LOGGER_OBJS = $(patsubst src/%.cpp, build/%.o, ${SOLAR_LOGGER_CPPS}) SOLAR_LOGGER_OBJS = $(patsubst src/%.cpp, build/%.o, ${SOLAR_LOGGER_CPPS})
@@ -13,10 +12,6 @@ SOLAR_SRV_CPPS = $(shell find src/solar-server/ -name *.cpp)
SOLAR_SRV_OBJS = $(patsubst src/%.cpp, build/%.o, ${SOLAR_SRV_CPPS}) SOLAR_SRV_OBJS = $(patsubst src/%.cpp, build/%.o, ${SOLAR_SRV_CPPS})
SOLAR_SRV_DEPS = $(patsubst src/%.cpp, build/%.d, ${SOLAR_SRV_CPPS}) SOLAR_SRV_DEPS = $(patsubst src/%.cpp, build/%.d, ${SOLAR_SRV_CPPS})
MIGRATOR_CPPS = $(shell find src/migrator/ -name *.cpp)
MIGRATOR_OBJS = $(patsubst src/%.cpp, build/%.o, ${MIGRATOR_CPPS})
MIGRATOR_DEPS = $(patsubst src/%.cpp, build/%.d, ${MIGRATOR_CPPS})
ELECT_LOGGER_CPPS = $(shell find src/electricity-logger/ -name *.cpp) ELECT_LOGGER_CPPS = $(shell find src/electricity-logger/ -name *.cpp)
ELECT_LOGGER_OBJS = $(patsubst src/%.cpp, build/%.o, ${ELECT_LOGGER_CPPS}) ELECT_LOGGER_OBJS = $(patsubst src/%.cpp, build/%.o, ${ELECT_LOGGER_CPPS})
ELECT_LOGGER_DEPS = $(patsubst src/%.cpp, build/%.d, ${ELECT_LOGGER_CPPS}) ELECT_LOGGER_DEPS = $(patsubst src/%.cpp, build/%.d, ${ELECT_LOGGER_CPPS})
@@ -27,7 +22,6 @@ ELECT_SRV_DEPS = $(patsubst src/%.cpp, build/%.d, ${ELECT_SRV_CPPS})
SOLAR_LOGGER_BINARY_PATH = bin/solar-logger SOLAR_LOGGER_BINARY_PATH = bin/solar-logger
SOLAR_SRV_BINARY_PATH = bin/solar-server SOLAR_SRV_BINARY_PATH = bin/solar-server
MIGRATOR_BINARY_PATH = bin/migrator
ELECT_LOGGER_BINARY_PATH = bin/electricity-logger ELECT_LOGGER_BINARY_PATH = bin/electricity-logger
ELECT_SRV_BINARY_PATH = bin/electricity-server ELECT_SRV_BINARY_PATH = bin/electricity-server
@@ -36,7 +30,7 @@ INSTALL_PATH = /usr/local/bin/
# Test variables # Test variables
SOLAR_TEST_DATABASE = solarpaneloutput.db SOLAR_TEST_DATABASE = solarpaneloutput.db
all: ${SOLAR_LOGGER_BINARY_PATH} ${SOLAR_SRV_BINARY_PATH} ${MIGRATOR_BINARY_PATH} ${ELECT_LOGGER_BINARY_PATH} ${ELECT_SRV_BINARY_PATH} all: ${SOLAR_LOGGER_BINARY_PATH} ${SOLAR_SRV_BINARY_PATH} ${ELECT_LOGGER_BINARY_PATH} ${ELECT_SRV_BINARY_PATH}
clean: clean:
-rm -rf bin/* build/* -rm -rf bin/* build/*
@@ -86,10 +80,6 @@ ${SOLAR_SRV_BINARY_PATH}: ${SOLAR_SRV_OBJS}
mkdir -p ${@D} mkdir -p ${@D}
${CC} $^ ${SERVER_LFLAGS} -o $@ ${CC} $^ ${SERVER_LFLAGS} -o $@
${MIGRATOR_BINARY_PATH}: ${MIGRATOR_OBJS}
mkdir -p ${@D}
${CC} $^ ${MIGRATOR_LFLAGS} -o $@
${ELECT_LOGGER_BINARY_PATH}: ${ELECT_LOGGER_OBJS} ${ELECT_LOGGER_BINARY_PATH}: ${ELECT_LOGGER_OBJS}
mkdir -p ${@D} mkdir -p ${@D}
${CC} $^ ${ELECTRICITY_LOGGER_LFLAGS} -o $@ ${CC} $^ ${ELECTRICITY_LOGGER_LFLAGS} -o $@

32
src/migrator/main.cpp
View File

@@ -1,32 +0,0 @@
#include <cstdio>
#include <migrator/migrations.hpp>
#include <tclap/CmdLine.h>
int main(int argc, char ** argv)
{
TCLAP::CmdLine cmd("migrator is a small program designed to run migrations", ' ', "1.0");
TCLAP::ValueArg<std::string> migrationArg("m", "migration", "", true, "", "Name of the migration to run");
cmd.add(migrationArg);
TCLAP::ValueArg<std::string> srcArg("s", "source", "", false, "", "Source file or directory to use");
cmd.add(srcArg);
TCLAP::ValueArg<std::string> dstArg("d", "destination", "", false, "", "Destination file or directory to use");
cmd.add(dstArg);
cmd.parse(argc, argv);
auto const & migrationToRun = migrationArg.getValue();
if(migrationToRun == "epochToDateTime")
{
return Migrations::EpochToDateTime(srcArg.getValue(), dstArg.getValue());
}
if(migrationToRun == "updateSummaryTable")
{
return Migrations::UpdateSummaryTable(srcArg.getValue(), dstArg.getValue());
}
std::printf("Unknown migration %s, aborting.\n", migrationToRun.c_str());
return 1;
}

91
src/migrator/migrations/epochtodatetime.cpp
View File

@@ -1,91 +0,0 @@
#include <ctime>
#include <migrator/transaction.hpp>
#include <sqlite3.h>
#include <sstream>
#include <string>
std::string ZeroPadTwoDigitNumber(int number)
{
auto result = std::to_string(number);
if(result.size() == 1)
{
return "0" + result;
}
return result;
}
int SourceCallback(void * data, int argc, char ** argv, char ** columnNames)
{
if(argc != 3)
{
std::puts("Expected source database to have 3 columns: DateTimeUtc, Watt and KilowattHour.");
return 1;
}
Transaction * const transactionPtr = reinterpret_cast<Transaction *>(data);
auto const epochTime = std::atol(argv[0]);
auto const dateTime = *std::gmtime(&epochTime);
std::stringstream insertStream;
insertStream << "INSERT INTO SolarPanelOutput VALUES(" << '\'' << (dateTime.tm_year + 1900) << '-'
<< ZeroPadTwoDigitNumber(dateTime.tm_mon + 1) << '-' << ZeroPadTwoDigitNumber(dateTime.tm_mday)
<< "','" << ZeroPadTwoDigitNumber(dateTime.tm_hour) << ':' << ZeroPadTwoDigitNumber(dateTime.tm_min)
<< ':' << ZeroPadTwoDigitNumber(dateTime.tm_sec) << "'," << argv[1] << ',' << argv[2] << ");";
transactionPtr->AddStatement(insertStream.str());
if(transactionPtr->StatementCount() > 1000)
{
return transactionPtr->Execute();
}
return 0;
}
namespace Migrations
{
int EpochToDateTime(std::string const & sourceDatabase, std::string const & destinationDatabase)
{
if(sourceDatabase == destinationDatabase)
{
std::puts("The EpochToDateTime is not meant to be run on the same database.");
return 1;
}
sqlite3 * source;
if(sqlite3_open(sourceDatabase.c_str(), &source))
{
std::printf("Error opening source database %s\n", sourceDatabase.c_str());
return 1;
}
sqlite3 * destination;
if(sqlite3_open(destinationDatabase.c_str(), &destination))
{
std::printf("Error opening destination database %s\n", destinationDatabase.c_str());
return 1;
}
Transaction transaction(destination);
auto const sqlResult
= sqlite3_exec(source, "SELECT * FROM SolarPanelOutput;", SourceCallback, &transaction, nullptr);
if(sqlResult)
{
std::printf("Error %i during insertion of records into destination database\n", sqlResult);
return 1;
}
auto const commitResult = transaction.Execute();
if(commitResult)
{
std::printf("Error %i when committing last transaction\n", commitResult);
return 1;
}
sqlite3_close(source);
sqlite3_close(destination);
return 0;
}
}

68
src/migrator/migrations/updatesummarytable.cpp
View File

@@ -1,68 +0,0 @@
#include <migrator/transaction.hpp>
#include <sqlite3.h>
#include <sstream>
namespace Migrations
{
int SourceCallback(void * data, int argc, char ** argv, char ** columnNames)
{
// Expect Date and KilowattHour
if(argc != 2)
{
std::puts("Wrong number of columns received");
return 1;
}
Transaction * transactionPtr = reinterpret_cast<Transaction *>(data);
std::stringstream insertStream;
insertStream << "INSERT INTO SolarPanelSummary(Date,KilowattHour) VALUES(" << '\'' << argv[0] << "'," << argv[1]
<< ')' << "ON CONFLICT(Date) DO UPDATE SET KilowattHour = excluded.KilowattHour;";
transactionPtr->AddStatement(insertStream.str());
return 0;
}
int UpdateSummaryTable(std::string const & sourceDatabase, std::string const & destinationDatabase)
{
sqlite3 * source;
if(sqlite3_open(sourceDatabase.c_str(), &source))
{
std::printf("Error opening source database %s\n", sourceDatabase.c_str());
return 1;
}
sqlite3 * destination;
if(sqlite3_open(destinationDatabase.c_str(), &destination))
{
std::printf("Error opening destination database %s\n", destinationDatabase.c_str());
return 1;
}
Transaction transaction(destination);
auto const sqlResult = sqlite3_exec(
source,
"SELECT Date, MAX(KilowattHour) AS KilowattHour FROM SolarPanelOutput GROUP BY Date;",
SourceCallback,
&transaction,
nullptr);
if(sqlResult)
{
std::printf("Error %i during fetching of source database records\n", sqlResult);
return 1;
}
std::printf("UpdateSummaryTable: Upserting %u records...\n", transaction.StatementCount());
auto const commitResult = transaction.Execute();
if(commitResult)
{
std::printf("Error %i when committing transaction\n", commitResult);
return 1;
}
sqlite3_close(source);
sqlite3_close(destination);
return 0;
}
}

35
src/migrator/transaction.cpp
View File

@@ -1,35 +0,0 @@
#include <migrator/transaction.hpp>
void Transaction::Reset()
{
queryStream.clear();
queryStream.str(std::string());
queryCount = 0u;
queryStream << "PRAGMA journal_mode = OFF;"
<< "BEGIN TRANSACTION;";
}
void Transaction::AddStatement(std::string const & statement)
{
++queryCount;
queryStream << statement;
}
unsigned Transaction::StatementCount() const { return queryCount; }
int Transaction::Execute()
{
queryStream << "COMMIT;";
auto const result = sqlite3_exec(destination, queryStream.str().c_str(), nullptr, nullptr, nullptr);
Reset();
return result;
}
Transaction::Transaction(sqlite3 * const databaseToInsertIn)
: destination(databaseToInsertIn), queryCount(0u), queryStream()
{
Reset();
}