WIP

home-data-collection-tools.sln

@@ -0,0 +1,53 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.002.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{A67575D5-676E-4A0C-A801-6AD47C23C9B1}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Solar.Api.Tests", "tests\Solar.Api.Tests\Solar.Api.Tests.csproj", "{38283E44-5C0D-4794-B497-43B83282C491}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{7BBFB17D-BEE6-45A8-993F-A92203EF54BE}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Electricity.Migrator", "src\Electricity.Migrator\Electricity.Migrator.csproj", "{3AA85834-D70A-4064-86DC-F30571FCAC32}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Electricity.Api", "src\Electricity.Api\Electricity.Api.csproj", "{B02C4DFC-1BA8-4D54-AD5B-14008F740B0F}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Solar.Api", "src\Solar.Api\Solar.Api.csproj", "{F5D8C125-9E59-4241-8ED3-9DAC84ABEF93}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{38283E44-5C0D-4794-B497-43B83282C491}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{38283E44-5C0D-4794-B497-43B83282C491}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{38283E44-5C0D-4794-B497-43B83282C491}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{38283E44-5C0D-4794-B497-43B83282C491}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3AA85834-D70A-4064-86DC-F30571FCAC32}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3AA85834-D70A-4064-86DC-F30571FCAC32}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3AA85834-D70A-4064-86DC-F30571FCAC32}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3AA85834-D70A-4064-86DC-F30571FCAC32}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B02C4DFC-1BA8-4D54-AD5B-14008F740B0F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B02C4DFC-1BA8-4D54-AD5B-14008F740B0F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B02C4DFC-1BA8-4D54-AD5B-14008F740B0F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B02C4DFC-1BA8-4D54-AD5B-14008F740B0F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F5D8C125-9E59-4241-8ED3-9DAC84ABEF93}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F5D8C125-9E59-4241-8ED3-9DAC84ABEF93}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F5D8C125-9E59-4241-8ED3-9DAC84ABEF93}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F5D8C125-9E59-4241-8ED3-9DAC84ABEF93}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{38283E44-5C0D-4794-B497-43B83282C491} = {A67575D5-676E-4A0C-A801-6AD47C23C9B1}
+		{3AA85834-D70A-4064-86DC-F30571FCAC32} = {7BBFB17D-BEE6-45A8-993F-A92203EF54BE}
+		{B02C4DFC-1BA8-4D54-AD5B-14008F740B0F} = {7BBFB17D-BEE6-45A8-993F-A92203EF54BE}
+		{F5D8C125-9E59-4241-8ED3-9DAC84ABEF93} = {7BBFB17D-BEE6-45A8-993F-A92203EF54BE}
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {BF72B2F1-30E0-4D62-8A8F-F7164218954B}
+	EndGlobalSection
+EndGlobal

src/Electricity.Api/Middlewares/IpFilterMiddleware.cs

@@ -0,0 +1,25 @@
+using System.Net;
+
+namespace Electricity.Api.Middlewares;
+
+internal sealed class IpFilterMiddleware(RequestDelegate next, IConfiguration configuration)
+{
+    private readonly RequestDelegate _next = next;
+    private readonly string AllowedHost = configuration["AllowedRequestHost"]
+        ?? throw new NullReferenceException($"Configuration value AllowedRequestHost is not available");
+
+    public async Task InvokeAsync(HttpContext context)
+    {
+        var selfIpAddress = (await Dns.GetHostEntryAsync(AllowedHost)).AddressList.FirstOrDefault()?.MapToIPv4().ToString();
+        if (selfIpAddress == null ||
+            !context.Request.Headers.TryGetValue("X-Real-IP", out var realIpHeaderValue) ||
+            selfIpAddress != realIpHeaderValue.ToString())
+        {
+            context.Response.StatusCode = (int)HttpStatusCode.Forbidden;
+            return;
+        }
+
+        // Call the next delegate/middleware in the pipeline.
+        await _next(context);
+    }
+}

src/Electricity.Api/Program.cs

@@ -33,16 +33,6 @@ builder.Services.AddDbContext<DatabaseContext>((DbContextOptionsBuilder builder)
     builder.UseSqlite($"Data Source={sqlite3DatabaseFilePath}");
 });
 
-builder.Services.AddCors(options =>
-{
-    options.AddPolicy(
-        name: "default",
-        policy =>
-        {
-            policy.WithOrigins("http://localhost:8080");
-        });
-});
-
 builder.Services.AddControllers()
     .AddJsonOptions(config =>
     {
@@ -62,6 +52,7 @@ if (app.Environment.IsDevelopment())
     app.UseCors("default");
 }
 
+app.UseMiddleware<Electricity.Api.Middlewares.IpFilterMiddleware>();
 app.UseAuthorization();
 
 app.MapControllers();

src/Electricity.Api/appsettings.json

@@ -5,5 +5,6 @@
       "Microsoft.AspNetCore": "Warning"
     }
   },
-  "AllowedHosts": "*"
+  "AllowedHosts": "*",
+  "AllowedRequestHost": "electricity.valkendaal.duckdns.org"
 }

src/Electricity.Migrator/.gitignore

@@ -0,0 +1,2 @@
+obj/
+bin/

src/Electricity.Migrator/DatabaseContext.cs

@@ -0,0 +1,45 @@
+using Electricity.Migrator.Entities;
+using Microsoft.EntityFrameworkCore;
+
+namespace Electricity.Migrator
+{
+    public partial class DatabaseContext : DbContext
+    {
+        public DatabaseContext()
+        {
+        }
+
+        public DatabaseContext(DbContextOptions<DatabaseContext> options)
+            : base(options)
+        {
+        }
+
+        public virtual DbSet<ElectricityLog> ElectricityLogs { get; set; } = null!;
+
+        protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
+        {
+            if (!optionsBuilder.IsConfigured)
+            {
+                optionsBuilder.UseSqlite("Data Source=test.db");
+            }
+        }
+
+        protected override void OnModelCreating(ModelBuilder modelBuilder)
+        {
+            modelBuilder.Entity<ElectricityLog>(entity =>
+            {
+                entity.HasNoKey();
+
+                entity.ToTable("ElectricityLog");
+
+                entity.HasIndex(e => e.Date, "idx_Date");
+
+                entity.HasIndex(e => e.TimeUtc, "idx_TimeUtc");
+            });
+
+            OnModelCreatingPartial(modelBuilder);
+        }
+
+        partial void OnModelCreatingPartial(ModelBuilder modelBuilder);
+    }
+}

src/Electricity.Migrator/Electricity.Migrator.csproj

@@ -0,0 +1,23 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="EFCore.BulkExtensions" Version="7.1.6" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.11">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="7.0.11" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
+    <PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="2.1.6" />
+    <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
+  </ItemGroup>
+
+</Project>

src/Electricity.Migrator/Entities/ElectricityLog.cs

@@ -0,0 +1,47 @@
+using System.Globalization;
+
+namespace Electricity.Migrator.Entities
+{
+    public partial class ElectricityLog
+    {
+        public string Date { get; set; } = null!;
+        public string TimeUtc { get; set; } = null!;
+        public double CurrentPowerUsage { get; set; }
+        public double TotalPowerConsumptionDay { get; set; }
+        public double TotalPowerConsumptionNight { get; set; }
+        public double CurrentPowerReturn { get; set; }
+        public double TotalPowerReturnDay { get; set; }
+        public double TotalPowerReturnNight { get; set; }
+        public long DayTarifEnabled { get; set; }
+        public double GasConsumptionInCubicMeters { get; set; }
+
+        public DateTime GetDateTime() => DateTime
+            .Parse(
+                $"{Date}T{TimeUtc}Z",
+                null,
+                System.Globalization.DateTimeStyles.AssumeUniversal)
+            .ToUniversalTime();
+
+        public static ElectricityLog CreateFromLine(string line)
+        {
+            // Example
+            // 0                   1    2     3     4 5      6      7         8             9
+            // 2018-03-30T10:25:08,0.08,69.47,52.62,0,20.519,56.158,day-tarif,180330102013S,188.593
+            var parts = line.Split(',');
+            var dateTimeParts = parts[0].Split('T');
+            return new ElectricityLog
+            {
+                Date = dateTimeParts[0],
+                TimeUtc = dateTimeParts[1],
+                CurrentPowerUsage = double.Parse(parts[1], CultureInfo.InvariantCulture),
+                TotalPowerConsumptionDay = double.Parse(parts[2], CultureInfo.InvariantCulture),
+                TotalPowerConsumptionNight = double.Parse(parts[3], CultureInfo.InvariantCulture),
+                CurrentPowerReturn = double.Parse(parts[4], CultureInfo.InvariantCulture),
+                TotalPowerReturnDay = double.Parse(parts[5], CultureInfo.InvariantCulture),
+                TotalPowerReturnNight = double.Parse(parts[6], CultureInfo.InvariantCulture),
+                DayTarifEnabled = parts[7] == "day-tarif" ? 1 : 0,
+                GasConsumptionInCubicMeters = double.Parse(parts[9], CultureInfo.InvariantCulture),
+            };
+        }
+    }
+}

src/Electricity.Migrator/MigratorHostedService.cs

@@ -0,0 +1,52 @@
+
+using EFCore.BulkExtensions;
+using Electricity.Migrator.Entities;
+using Electricity.Migrator.Options;
+using Microsoft.Extensions.Hosting;
+
+namespace Electricity.Migrator;
+
+internal class MigratorHostedService : IHostedService
+{
+    private readonly DatabaseContext database;
+    private readonly LogFileOption logFileOption;
+
+    public MigratorHostedService(DatabaseContext database, LogFileOption logFileOption)
+    {
+        this.database = database;
+        this.logFileOption = logFileOption;
+    }
+
+    public async Task StartAsync(CancellationToken cancellationToken)
+    {
+        database.ChangeTracker.AutoDetectChangesEnabled = false;
+        await database.Database.EnsureCreatedAsync();
+
+        foreach (var directory in Directory.EnumerateDirectories(logFileOption.Path).OrderBy(d => d))
+        {
+            foreach (var file in Directory.EnumerateFiles($"{logFileOption.Path}/{directory}").OrderBy(f => f))
+            {
+                var entities = new List<ElectricityLog>();
+                foreach (var line in await File.ReadAllLinesAsync(file))
+                {
+                    entities.Add(ElectricityLog.CreateFromLine(line));
+
+                    if (entities.Count >= 1000)
+                    {
+                        await database.BulkInsertAsync(entities);
+                        entities.Clear();
+                    }
+                }
+
+                await database.BulkInsertAsync(entities);
+            }
+
+            await database.SaveChangesAsync();
+        }
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        return Task.CompletedTask;
+    }
+}

src/Electricity.Migrator/Options/LogFileOption.cs

@@ -0,0 +1,3 @@
+namespace Electricity.Migrator.Options;
+
+internal record LogFileOption(string Path);

src/Electricity.Migrator/Program.cs

@@ -0,0 +1,45 @@
+using System.CommandLine;
+using Electricity.Migrator;
+using Electricity.Migrator.Options;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+
+var rootCommand = new RootCommand("This is a migrator tool to migrate file logs to an Sqlite3 database");
+
+var connectionStringArgument = new Option<string>(
+    name: "--connection-string",
+    description: "Filepath to the Sqlite3 database file (*.db) to create");
+rootCommand.AddOption(connectionStringArgument);
+
+var logFilePathArgument = new Option<string>(
+    name: "--log-files",
+    description: "Filepath directory containing the year directories of the log files");
+rootCommand.AddOption(logFilePathArgument);
+
+var result = rootCommand.Parse(args);
+
+if (result.Errors.Any())
+{
+    foreach (var error in result.Errors)
+    {
+        Console.WriteLine(error.Message);
+    }
+    return 1;
+}
+
+var builder = Host.CreateApplicationBuilder(args);
+builder.Services.AddDbContext<DatabaseContext>((DbContextOptionsBuilder builder) =>
+{
+    builder.UseSqlite($"Data Source={result.GetValueForOption(connectionStringArgument)}");
+});
+
+builder.Services.AddSingleton(new LogFileOption(result.GetValueForOption(logFilePathArgument)));
+
+builder.Services.AddHostedService<MigratorHostedService>();
+
+var host = builder.Build();
+
+host.Run();
+
+return 0;

tests/Solar.Api.Tests/.gitignore

@@ -0,0 +1,2 @@
+bin/
+obj/

tests/Solar.Api.Tests/Controllers/SolarLogController.cs

@@ -0,0 +1,20 @@
+using Solar.Api.Models;
+
+namespace Solar.Api.Tests.Controllers;
+
+[Collection("Default")]
+public class SolarLogController
+{
+    private readonly DefaultWebApplicationFactory applicationFactory;
+
+    public SolarLogController(DefaultWebApplicationFactory applicationFactory)
+    {
+        this.applicationFactory = applicationFactory;
+    }
+
+    [Fact]
+    public void GetDayDetails_returns_for_existing_data()
+    {
+        var client = applicationFactory.CreateClient();
+    }
+}

tests/Solar.Api.Tests/DefaultCollectionDefinition.cs

@@ -0,0 +1,6 @@
+namespace Solar.Api.Tests;
+
+[CollectionDefinition("Default")]
+public class DefaultCollectionDefinition : ICollectionFixture<DefaultWebApplicationFactory>
+{
+}

tests/Solar.Api.Tests/DefaultWebApplicationFactory.cs

@@ -0,0 +1,30 @@
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Mvc.Testing;
+using Microsoft.AspNetCore.TestHost;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Solar.Api.Tests;
+
+public class DefaultWebApplicationFactory : WebApplicationFactory<Program>
+{
+    protected override void ConfigureWebHost(IWebHostBuilder builder)
+    {
+        builder.ConfigureTestServices(services =>
+        {
+            services.AddDbContext<DatabaseContext>(options =>
+            {
+                options.UseSqlite("Data Source=test.db");
+            });
+        });
+
+        builder.ConfigureServices(services =>
+        {
+            using var provider = services.BuildServiceProvider();
+            using var scope = provider.CreateScope();
+            using var databaseContext = scope.ServiceProvider.GetRequiredService<DatabaseContext>();
+
+            databaseContext.Database.EnsureCreated();
+        });
+    }
+}

tests/Solar.Api.Tests/Extensions/HttpClientExtensions.cs

@@ -0,0 +1,18 @@
+using System.Text.Json;
+using Shouldly;
+
+namespace Solar.Api.Tests.Extensions;
+
+public static class HttpClientExtensions
+{
+    public static async Task<T> FetchJson<T>(this HttpClient client, string path) where T : class
+    {
+        var response = await client.GetAsync(path);
+        response.EnsureSuccessStatusCode();
+
+        var result = JsonSerializer.Deserialize<T>(await response.Content.ReadAsStringAsync());
+        result.ShouldNotBeNull();
+
+        return result;
+    }
+}

tests/Solar.Api.Tests/Solar.Api.Tests.csproj

@@ -0,0 +1,30 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="7.0.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
+    <PackageReference Include="Shouldly" Version="4.1.0" />
+    <PackageReference Include="xunit" Version="2.4.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="coverlet.collector" Version="3.1.2">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Solar.Api\Solar.Api.csproj" />
+  </ItemGroup>
+
+</Project>

tests/Solar.Api.Tests/Usings.cs

@@ -0,0 +1,2 @@
+global using Xunit;
+global using Solar.Api.Tests.Extensions;