diff --git a/home-data-collection-tools.sln b/home-data-collection-tools.sln new file mode 100644 index 0000000..2e5e64d --- /dev/null +++ b/home-data-collection-tools.sln @@ -0,0 +1,53 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.002.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{A67575D5-676E-4A0C-A801-6AD47C23C9B1}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Solar.Api.Tests", "tests\Solar.Api.Tests\Solar.Api.Tests.csproj", "{38283E44-5C0D-4794-B497-43B83282C491}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{7BBFB17D-BEE6-45A8-993F-A92203EF54BE}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Electricity.Migrator", "src\Electricity.Migrator\Electricity.Migrator.csproj", "{3AA85834-D70A-4064-86DC-F30571FCAC32}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Electricity.Api", "src\Electricity.Api\Electricity.Api.csproj", "{B02C4DFC-1BA8-4D54-AD5B-14008F740B0F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Solar.Api", "src\Solar.Api\Solar.Api.csproj", "{F5D8C125-9E59-4241-8ED3-9DAC84ABEF93}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {38283E44-5C0D-4794-B497-43B83282C491}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {38283E44-5C0D-4794-B497-43B83282C491}.Debug|Any CPU.Build.0 = Debug|Any CPU + {38283E44-5C0D-4794-B497-43B83282C491}.Release|Any CPU.ActiveCfg = Release|Any CPU + {38283E44-5C0D-4794-B497-43B83282C491}.Release|Any CPU.Build.0 = Release|Any CPU + {3AA85834-D70A-4064-86DC-F30571FCAC32}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3AA85834-D70A-4064-86DC-F30571FCAC32}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3AA85834-D70A-4064-86DC-F30571FCAC32}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3AA85834-D70A-4064-86DC-F30571FCAC32}.Release|Any CPU.Build.0 = Release|Any CPU + {B02C4DFC-1BA8-4D54-AD5B-14008F740B0F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B02C4DFC-1BA8-4D54-AD5B-14008F740B0F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B02C4DFC-1BA8-4D54-AD5B-14008F740B0F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B02C4DFC-1BA8-4D54-AD5B-14008F740B0F}.Release|Any CPU.Build.0 = Release|Any CPU + {F5D8C125-9E59-4241-8ED3-9DAC84ABEF93}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F5D8C125-9E59-4241-8ED3-9DAC84ABEF93}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F5D8C125-9E59-4241-8ED3-9DAC84ABEF93}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F5D8C125-9E59-4241-8ED3-9DAC84ABEF93}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {38283E44-5C0D-4794-B497-43B83282C491} = {A67575D5-676E-4A0C-A801-6AD47C23C9B1} + {3AA85834-D70A-4064-86DC-F30571FCAC32} = {7BBFB17D-BEE6-45A8-993F-A92203EF54BE} + {B02C4DFC-1BA8-4D54-AD5B-14008F740B0F} = {7BBFB17D-BEE6-45A8-993F-A92203EF54BE} + {F5D8C125-9E59-4241-8ED3-9DAC84ABEF93} = {7BBFB17D-BEE6-45A8-993F-A92203EF54BE} + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {BF72B2F1-30E0-4D62-8A8F-F7164218954B} + EndGlobalSection +EndGlobal diff --git a/src/Electricity.Api/Middlewares/IpFilterMiddleware.cs b/src/Electricity.Api/Middlewares/IpFilterMiddleware.cs new file mode 100644 index 0000000..bd865a5 --- /dev/null +++ b/src/Electricity.Api/Middlewares/IpFilterMiddleware.cs @@ -0,0 +1,25 @@ +using System.Net; + +namespace Electricity.Api.Middlewares; + +internal sealed class IpFilterMiddleware(RequestDelegate next, IConfiguration configuration) +{ + private readonly RequestDelegate _next = next; + private readonly string AllowedHost = configuration["AllowedRequestHost"] + ?? throw new NullReferenceException($"Configuration value AllowedRequestHost is not available"); + + public async Task InvokeAsync(HttpContext context) + { + var selfIpAddress = (await Dns.GetHostEntryAsync(AllowedHost)).AddressList.FirstOrDefault()?.MapToIPv4().ToString(); + if (selfIpAddress == null || + !context.Request.Headers.TryGetValue("X-Real-IP", out var realIpHeaderValue) || + selfIpAddress != realIpHeaderValue.ToString()) + { + context.Response.StatusCode = (int)HttpStatusCode.Forbidden; + return; + } + + // Call the next delegate/middleware in the pipeline. + await _next(context); + } +} \ No newline at end of file diff --git a/src/Electricity.Api/Program.cs b/src/Electricity.Api/Program.cs index 8888e83..4e12742 100644 --- a/src/Electricity.Api/Program.cs +++ b/src/Electricity.Api/Program.cs @@ -33,16 +33,6 @@ builder.Services.AddDbContext((DbContextOptionsBuilder builder) builder.UseSqlite($"Data Source={sqlite3DatabaseFilePath}"); }); -builder.Services.AddCors(options => -{ - options.AddPolicy( - name: "default", - policy => - { - policy.WithOrigins("http://localhost:8080"); - }); -}); - builder.Services.AddControllers() .AddJsonOptions(config => { @@ -62,6 +52,7 @@ if (app.Environment.IsDevelopment()) app.UseCors("default"); } +app.UseMiddleware(); app.UseAuthorization(); app.MapControllers(); diff --git a/src/Electricity.Api/appsettings.json b/src/Electricity.Api/appsettings.json index 4d56694..5854828 100644 --- a/src/Electricity.Api/appsettings.json +++ b/src/Electricity.Api/appsettings.json @@ -5,5 +5,6 @@ "Microsoft.AspNetCore": "Warning" } }, - "AllowedHosts": "*" + "AllowedHosts": "*", + "AllowedRequestHost": "electricity.valkendaal.duckdns.org" } diff --git a/src/Electricity.Migrator/.gitignore b/src/Electricity.Migrator/.gitignore new file mode 100644 index 0000000..d86ba9f --- /dev/null +++ b/src/Electricity.Migrator/.gitignore @@ -0,0 +1,2 @@ +obj/ +bin/ \ No newline at end of file diff --git a/src/Electricity.Migrator/DatabaseContext.cs b/src/Electricity.Migrator/DatabaseContext.cs new file mode 100644 index 0000000..6fda4cd --- /dev/null +++ b/src/Electricity.Migrator/DatabaseContext.cs @@ -0,0 +1,45 @@ +using Electricity.Migrator.Entities; +using Microsoft.EntityFrameworkCore; + +namespace Electricity.Migrator +{ + public partial class DatabaseContext : DbContext + { + public DatabaseContext() + { + } + + public DatabaseContext(DbContextOptions options) + : base(options) + { + } + + public virtual DbSet ElectricityLogs { get; set; } = null!; + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + { + if (!optionsBuilder.IsConfigured) + { + optionsBuilder.UseSqlite("Data Source=test.db"); + } + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.HasNoKey(); + + entity.ToTable("ElectricityLog"); + + entity.HasIndex(e => e.Date, "idx_Date"); + + entity.HasIndex(e => e.TimeUtc, "idx_TimeUtc"); + }); + + OnModelCreatingPartial(modelBuilder); + } + + partial void OnModelCreatingPartial(ModelBuilder modelBuilder); + } +} diff --git a/src/Electricity.Migrator/Electricity.Migrator.csproj b/src/Electricity.Migrator/Electricity.Migrator.csproj new file mode 100644 index 0000000..38e4cc4 --- /dev/null +++ b/src/Electricity.Migrator/Electricity.Migrator.csproj @@ -0,0 +1,23 @@ + + + + Exe + net7.0 + enable + enable + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + diff --git a/src/Electricity.Migrator/Entities/ElectricityLog.cs b/src/Electricity.Migrator/Entities/ElectricityLog.cs new file mode 100644 index 0000000..31965b0 --- /dev/null +++ b/src/Electricity.Migrator/Entities/ElectricityLog.cs @@ -0,0 +1,47 @@ +using System.Globalization; + +namespace Electricity.Migrator.Entities +{ + public partial class ElectricityLog + { + public string Date { get; set; } = null!; + public string TimeUtc { get; set; } = null!; + public double CurrentPowerUsage { get; set; } + public double TotalPowerConsumptionDay { get; set; } + public double TotalPowerConsumptionNight { get; set; } + public double CurrentPowerReturn { get; set; } + public double TotalPowerReturnDay { get; set; } + public double TotalPowerReturnNight { get; set; } + public long DayTarifEnabled { get; set; } + public double GasConsumptionInCubicMeters { get; set; } + + public DateTime GetDateTime() => DateTime + .Parse( + $"{Date}T{TimeUtc}Z", + null, + System.Globalization.DateTimeStyles.AssumeUniversal) + .ToUniversalTime(); + + public static ElectricityLog CreateFromLine(string line) + { + // Example + // 0 1 2 3 4 5 6 7 8 9 + // 2018-03-30T10:25:08,0.08,69.47,52.62,0,20.519,56.158,day-tarif,180330102013S,188.593 + var parts = line.Split(','); + var dateTimeParts = parts[0].Split('T'); + return new ElectricityLog + { + Date = dateTimeParts[0], + TimeUtc = dateTimeParts[1], + CurrentPowerUsage = double.Parse(parts[1], CultureInfo.InvariantCulture), + TotalPowerConsumptionDay = double.Parse(parts[2], CultureInfo.InvariantCulture), + TotalPowerConsumptionNight = double.Parse(parts[3], CultureInfo.InvariantCulture), + CurrentPowerReturn = double.Parse(parts[4], CultureInfo.InvariantCulture), + TotalPowerReturnDay = double.Parse(parts[5], CultureInfo.InvariantCulture), + TotalPowerReturnNight = double.Parse(parts[6], CultureInfo.InvariantCulture), + DayTarifEnabled = parts[7] == "day-tarif" ? 1 : 0, + GasConsumptionInCubicMeters = double.Parse(parts[9], CultureInfo.InvariantCulture), + }; + } + } +} diff --git a/src/Electricity.Migrator/MigratorHostedService.cs b/src/Electricity.Migrator/MigratorHostedService.cs new file mode 100644 index 0000000..bffb883 --- /dev/null +++ b/src/Electricity.Migrator/MigratorHostedService.cs @@ -0,0 +1,52 @@ + +using EFCore.BulkExtensions; +using Electricity.Migrator.Entities; +using Electricity.Migrator.Options; +using Microsoft.Extensions.Hosting; + +namespace Electricity.Migrator; + +internal class MigratorHostedService : IHostedService +{ + private readonly DatabaseContext database; + private readonly LogFileOption logFileOption; + + public MigratorHostedService(DatabaseContext database, LogFileOption logFileOption) + { + this.database = database; + this.logFileOption = logFileOption; + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + database.ChangeTracker.AutoDetectChangesEnabled = false; + await database.Database.EnsureCreatedAsync(); + + foreach (var directory in Directory.EnumerateDirectories(logFileOption.Path).OrderBy(d => d)) + { + foreach (var file in Directory.EnumerateFiles($"{logFileOption.Path}/{directory}").OrderBy(f => f)) + { + var entities = new List(); + foreach (var line in await File.ReadAllLinesAsync(file)) + { + entities.Add(ElectricityLog.CreateFromLine(line)); + + if (entities.Count >= 1000) + { + await database.BulkInsertAsync(entities); + entities.Clear(); + } + } + + await database.BulkInsertAsync(entities); + } + + await database.SaveChangesAsync(); + } + } + + public Task StopAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } +} \ No newline at end of file diff --git a/src/Electricity.Migrator/Options/LogFileOption.cs b/src/Electricity.Migrator/Options/LogFileOption.cs new file mode 100644 index 0000000..33aa416 --- /dev/null +++ b/src/Electricity.Migrator/Options/LogFileOption.cs @@ -0,0 +1,3 @@ +namespace Electricity.Migrator.Options; + +internal record LogFileOption(string Path); \ No newline at end of file diff --git a/src/Electricity.Migrator/Program.cs b/src/Electricity.Migrator/Program.cs new file mode 100644 index 0000000..3e5e386 --- /dev/null +++ b/src/Electricity.Migrator/Program.cs @@ -0,0 +1,45 @@ +using System.CommandLine; +using Electricity.Migrator; +using Electricity.Migrator.Options; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +var rootCommand = new RootCommand("This is a migrator tool to migrate file logs to an Sqlite3 database"); + +var connectionStringArgument = new Option( + name: "--connection-string", + description: "Filepath to the Sqlite3 database file (*.db) to create"); +rootCommand.AddOption(connectionStringArgument); + +var logFilePathArgument = new Option( + name: "--log-files", + description: "Filepath directory containing the year directories of the log files"); +rootCommand.AddOption(logFilePathArgument); + +var result = rootCommand.Parse(args); + +if (result.Errors.Any()) +{ + foreach (var error in result.Errors) + { + Console.WriteLine(error.Message); + } + return 1; +} + +var builder = Host.CreateApplicationBuilder(args); +builder.Services.AddDbContext((DbContextOptionsBuilder builder) => +{ + builder.UseSqlite($"Data Source={result.GetValueForOption(connectionStringArgument)}"); +}); + +builder.Services.AddSingleton(new LogFileOption(result.GetValueForOption(logFilePathArgument))); + +builder.Services.AddHostedService(); + +var host = builder.Build(); + +host.Run(); + +return 0; \ No newline at end of file diff --git a/tests/Solar.Api.Tests/.gitignore b/tests/Solar.Api.Tests/.gitignore new file mode 100644 index 0000000..cbbd0b5 --- /dev/null +++ b/tests/Solar.Api.Tests/.gitignore @@ -0,0 +1,2 @@ +bin/ +obj/ \ No newline at end of file diff --git a/tests/Solar.Api.Tests/Controllers/SolarLogController.cs b/tests/Solar.Api.Tests/Controllers/SolarLogController.cs new file mode 100644 index 0000000..56015bf --- /dev/null +++ b/tests/Solar.Api.Tests/Controllers/SolarLogController.cs @@ -0,0 +1,20 @@ +using Solar.Api.Models; + +namespace Solar.Api.Tests.Controllers; + +[Collection("Default")] +public class SolarLogController +{ + private readonly DefaultWebApplicationFactory applicationFactory; + + public SolarLogController(DefaultWebApplicationFactory applicationFactory) + { + this.applicationFactory = applicationFactory; + } + + [Fact] + public void GetDayDetails_returns_for_existing_data() + { + var client = applicationFactory.CreateClient(); + } +} \ No newline at end of file diff --git a/tests/Solar.Api.Tests/DefaultCollectionDefinition.cs b/tests/Solar.Api.Tests/DefaultCollectionDefinition.cs new file mode 100644 index 0000000..838c3d3 --- /dev/null +++ b/tests/Solar.Api.Tests/DefaultCollectionDefinition.cs @@ -0,0 +1,6 @@ +namespace Solar.Api.Tests; + +[CollectionDefinition("Default")] +public class DefaultCollectionDefinition : ICollectionFixture +{ +} \ No newline at end of file diff --git a/tests/Solar.Api.Tests/DefaultWebApplicationFactory.cs b/tests/Solar.Api.Tests/DefaultWebApplicationFactory.cs new file mode 100644 index 0000000..1b81612 --- /dev/null +++ b/tests/Solar.Api.Tests/DefaultWebApplicationFactory.cs @@ -0,0 +1,30 @@ +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; + +namespace Solar.Api.Tests; + +public class DefaultWebApplicationFactory : WebApplicationFactory +{ + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.ConfigureTestServices(services => + { + services.AddDbContext(options => + { + options.UseSqlite("Data Source=test.db"); + }); + }); + + builder.ConfigureServices(services => + { + using var provider = services.BuildServiceProvider(); + using var scope = provider.CreateScope(); + using var databaseContext = scope.ServiceProvider.GetRequiredService(); + + databaseContext.Database.EnsureCreated(); + }); + } +} \ No newline at end of file diff --git a/tests/Solar.Api.Tests/Extensions/HttpClientExtensions.cs b/tests/Solar.Api.Tests/Extensions/HttpClientExtensions.cs new file mode 100644 index 0000000..31fe0d2 --- /dev/null +++ b/tests/Solar.Api.Tests/Extensions/HttpClientExtensions.cs @@ -0,0 +1,18 @@ +using System.Text.Json; +using Shouldly; + +namespace Solar.Api.Tests.Extensions; + +public static class HttpClientExtensions +{ + public static async Task FetchJson(this HttpClient client, string path) where T : class + { + var response = await client.GetAsync(path); + response.EnsureSuccessStatusCode(); + + var result = JsonSerializer.Deserialize(await response.Content.ReadAsStringAsync()); + result.ShouldNotBeNull(); + + return result; + } +} \ No newline at end of file diff --git a/tests/Solar.Api.Tests/Solar.Api.Tests.csproj b/tests/Solar.Api.Tests/Solar.Api.Tests.csproj new file mode 100644 index 0000000..9c8295b --- /dev/null +++ b/tests/Solar.Api.Tests/Solar.Api.Tests.csproj @@ -0,0 +1,30 @@ + + + + net7.0 + enable + enable + + false + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/tests/Solar.Api.Tests/Usings.cs b/tests/Solar.Api.Tests/Usings.cs new file mode 100644 index 0000000..8a1f554 --- /dev/null +++ b/tests/Solar.Api.Tests/Usings.cs @@ -0,0 +1,2 @@ +global using Xunit; +global using Solar.Api.Tests.Extensions; \ No newline at end of file diff --git a/tests/Solar.Api.Tests/test.db b/tests/Solar.Api.Tests/test.db new file mode 100644 index 0000000..0b383b7 Binary files /dev/null and b/tests/Solar.Api.Tests/test.db differ