From d0a7e7ec880e515869c7baf1331c0bc54a7e4f99 Mon Sep 17 00:00:00 2001 From: lelgenio Date: Sat, 22 Jun 2024 13:42:24 -0300 Subject: [PATCH] deploy config --- .gitignore | 2 + Cargo.lock | 36 +++++++++-- Cargo.toml | 4 +- flake.lock | 21 +++++++ flake.nix | 118 +++++++++++++++++++++++++++++++---- src/controllers/home.rs | 50 ++++++++++++--- src/lib.rs | 23 +++++-- src/main.rs | 2 +- src/sources/mod.rs | 2 +- src/sources/wikipedia/mod.rs | 12 ++-- 10 files changed, 231 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 2d5df85..3aebaec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ /target .direnv +nixos.qcow2 +result diff --git a/Cargo.lock b/Cargo.lock index 8c819da..ee97cb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -780,6 +780,15 @@ dependencies = [ "tendril", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "matchit" version = "0.7.3" @@ -1288,8 +1297,17 @@ checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -1300,9 +1318,15 @@ checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.4", ] +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.4" @@ -1966,10 +1990,14 @@ version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", ] @@ -2092,7 +2120,7 @@ dependencies = [ ] [[package]] -name = "warthunder-confidential-document-leak-counter" +name = "warthunder-leak-counter" version = "0.1.0" dependencies = [ "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 220efef..9be4596 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "warthunder-confidential-document-leak-counter" +name = "warthunder-leak-counter" version = "0.1.0" edition = "2021" @@ -17,5 +17,5 @@ time = "0.3.36" tokio = { version = "1.38.0", features = ["full"] } tower-http = { version = "0.5.2", features = ["trace", "fs"] } tracing = "0.1.40" -tracing-subscriber = "0.3.18" +tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } diff --git a/flake.lock b/flake.lock index 9cf75f9..f1184c1 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,25 @@ { "nodes": { + "crane": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1718730147, + "narHash": "sha256-QmD6B6FYpuoCqu6ZuPJH896ItNquDkn0ulQlOn4ykN8=", + "owner": "ipetkov", + "repo": "crane", + "rev": "32c21c29b034d0a93fdb2379d6fabc40fc3d0e6c", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -36,6 +56,7 @@ }, "root": { "inputs": { + "crane": "crane", "flake-utils": "flake-utils", "nixpkgs": "nixpkgs" } diff --git a/flake.nix b/flake.nix index 2be098d..ed68ea8 100644 --- a/flake.nix +++ b/flake.nix @@ -3,6 +3,11 @@ nixpkgs.url = "github:NixOS/nixpkgs/release-24.05"; flake-utils.url = "github:numtide/flake-utils"; + + crane = { + url = "github:ipetkov/crane"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; outputs = @@ -10,6 +15,7 @@ self, nixpkgs, flake-utils, + crane, ... }: flake-utils.lib.eachDefaultSystem ( @@ -17,22 +23,110 @@ let pkgs = import nixpkgs { inherit system; }; inherit (pkgs) lib; + + craneLib = crane.mkLib pkgs; + + commonArgs = { + src = craneLib.cleanCargoSource ./.; + strictDeps = true; + nativeBuildInputs = with pkgs; [ pkg-config ]; + buildInputs = with pkgs; [ openssl ]; + }; + + cargoArtifacts = craneLib.buildDepsOnly commonArgs; + + my-crate = craneLib.buildPackage ( + commonArgs + // { + src = ./.; # Allow access to assets, like ./templates + inherit cargoArtifacts; + meta.mainProgram = "warthunder-leak-counter"; + } + ); in { - devShells.default = pkgs.mkShell { - nativeBuildInputs = with pkgs; [ - rustc - cargo - rustfmt - rust-analyzer - clippy - cargo-feature - cargo-watch - pkg-config - openssl - curl + checks = { + my-crate-fmt = craneLib.cargoFmt { inherit (commonArgs) src; }; + }; + + packages.default = my-crate; + + nixosModules.default = + { pkgs, config, ... }: + let + cfg = config.services.warthunder-leak-counter; + in + { + options.services.warthunder-leak-counter = { + enable = lib.mkEnableOption "Enable Warthunder Leak Counter"; + + staticDir = lib.mkOption { + default = toString ./static; + type = lib.types.str; + }; + port = lib.mkOption { + type = lib.types.port; + default = 6263; + }; + }; + config = lib.mkIf cfg.enable { + systemd.services.warthunder-leak-counter = { + script = lib.getExe my-crate; + + environment = { + WARTHUNDER_LEAK_SERVE_PORT = toString cfg.port; + WARTHUNDER_LEAK_STATIC_DIR = cfg.staticDir; + }; + + after = [ "network.target" ]; + wantedBy = [ "network.target" ]; + }; + }; + }; + + packages.nixosConfigurations.test-server = nixpkgs.lib.nixosSystem { + inherit system; + modules = [ + self.nixosModules.${system}.default + (nixpkgs + "/nixos/modules/virtualisation/qemu-vm.nix") + ( + { config, ... }: + { + services.warthunder-leak-counter.enable = true; + users.users.root.password = "root"; + networking.firewall.enable = false; + virtualisation.forwardPorts = [ + { + from = "host"; + host.port = 8888; + guest.port = config.services.warthunder-leak-counter.port; + } + ]; + system.stateVersion = "24.05"; + } + ) ]; }; + + devShells.default = pkgs.mkShell { + inherit (commonArgs) buildInputs; + + nativeBuildInputs = + with pkgs; + ( + [ + rustc + cargo + rustfmt + rust-analyzer + clippy + cargo-feature + cargo-watch + curl + ] + ++ commonArgs.nativeBuildInputs + ); + }; } ); } diff --git a/src/controllers/home.rs b/src/controllers/home.rs index e5fc497..791bb66 100644 --- a/src/controllers/home.rs +++ b/src/controllers/home.rs @@ -1,7 +1,8 @@ use askama::Template; +use axum::extract::State; use time::Date; -use crate::sources; +use crate::{sources, AppState}; #[derive(Template)] #[template(path = "index.html")] @@ -22,18 +23,53 @@ impl TimeSince { } #[axum::debug_handler] -pub async fn get() -> HomeTemplate { +pub async fn get(state: State) -> HomeTemplate { let mut t = vec![]; for source in sources::sources() { let url = source.url(); - let Ok(res) = (reqwest::get(url)).await else { - tracing::error!("fetch error"); - continue; + + let mut cache = state.0.get_cache.lock().await; + let now = time::OffsetDateTime::now_utc(); + + let needs_update = match cache.get(&url) { + None => { + tracing::info!("Value is not present in cache"); + true + } + Some((cached_time, _)) => { + let other_day = cached_time.to_julian_day() != now.to_julian_day(); + let other_hour = cached_time.hour() != now.hour(); + + if other_day { + tracing::info!("Value is from another day"); + } + if other_hour { + tracing::info!("Value is from another hour"); + } + + other_day || other_hour + } }; - let Ok(text) = res.text().await else { - tracing::error!("fetch decode text error"); + if needs_update { + tracing::info!("Need update cache"); + let Ok(res) = (reqwest::get(url.clone())).await else { + tracing::error!("fetch error"); + continue; + }; + + let Ok(text) = res.text().await else { + tracing::error!("fetch decode text error"); + continue; + }; + + tracing::info!("Cache updated"); + cache.insert(url.clone(), (now, text)); + } + + let Some((_, text)) = cache.get(&url) else { + tracing::error!("filling cache error"); continue; }; diff --git a/src/lib.rs b/src/lib.rs index bef80b4..1061f1c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,8 +1,9 @@ -use std::{future::Future, pin::Pin}; +use std::{collections::HashMap, future::Future, pin::Pin, sync::Arc}; use anyhow::Result; use axum::{routing::get, Router}; -use tokio::net::TcpListener; +use time::OffsetDateTime; +use tokio::{net::TcpListener, sync::Mutex}; use tower_http::services::ServeDir; mod controllers; @@ -11,7 +12,12 @@ mod sources; fn routes() -> Router { Router::new() .route("/", get(controllers::home::get)) - .fallback_service(ServeDir::new("./static")) + .with_state(AppState::default()) +} + +#[derive(Default, Clone)] +pub struct AppState { + get_cache: Arc>>, } pub struct Config { @@ -26,7 +32,11 @@ pub struct RunningServer { pub async fn run(config: Config) -> Result { setup_tracing(); - let router = routes().layer(tower_http::trace::TraceLayer::new_for_http()); + let static_dir = std::env::var("WARTHUNDER_LEAK_STATIC_DIR").unwrap_or("./static".to_string()); + + let router = routes() + .fallback_service(ServeDir::new(static_dir)) + .layer(tower_http::trace::TraceLayer::new_for_http()); let tcp_listener = TcpListener::bind(format!("0.0.0.0:{}", config.port)).await?; @@ -44,7 +54,12 @@ pub async fn run(config: Config) -> Result { pub fn setup_tracing() { use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; + let log_filter = std::env::var("WARTHUNDER_LOG").unwrap_or_else(|_| "warthunder_leak_counter=debug,warn".into()); + + eprintln!("RUST_LOG: {log_filter}"); + tracing_subscriber::registry() + .with(tracing_subscriber::EnvFilter::new(log_filter)) .with(tracing_subscriber::fmt::layer()) .try_init() .ok(); diff --git a/src/main.rs b/src/main.rs index 1401ddb..86c4feb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use warthunder_confidential_document_leak_counter::{run, Config}; +use warthunder_leak_counter::{run, Config}; #[tokio::main] async fn main() -> Result<()> { diff --git a/src/sources/mod.rs b/src/sources/mod.rs index 9f8aff0..7e851f6 100644 --- a/src/sources/mod.rs +++ b/src/sources/mod.rs @@ -6,7 +6,7 @@ pub trait Source { /// Return the URL to query fn url(&self) -> String; /// Given the content of the url figure out the date of the latest leak - fn latest_leak(&self, html: String) -> Result; + fn latest_leak(&self, html: &str) -> Result; } pub fn sources() -> Vec> { diff --git a/src/sources/wikipedia/mod.rs b/src/sources/wikipedia/mod.rs index 515df19..e2327ee 100644 --- a/src/sources/wikipedia/mod.rs +++ b/src/sources/wikipedia/mod.rs @@ -1,4 +1,4 @@ -use std::{str::FromStr, time::Instant}; +use std::{str::FromStr}; use super::Source; use anyhow::{bail, Context, Result}; @@ -12,8 +12,8 @@ impl Source for Wikipedia { "https://en.wikipedia.org/wiki/War_Thunder".to_string() } - fn latest_leak(&self, html: String) -> Result { - let soup = soup::Soup::new(&html); + fn latest_leak(&self, html: &str) -> Result { + let soup = soup::Soup::new(html); let tables = soup.tag("table").find_all(); @@ -82,7 +82,7 @@ fn parse_wikipedia_date(text: &str) -> Result { fn test_wikipedia_html_parse() { let html = std::fs::read_to_string("./data/wikipedia.html").unwrap(); - let real = Wikipedia.latest_leak(html).unwrap(); + let real = Wikipedia.latest_leak(&html).unwrap(); let expected = time::Date::from_calendar_date(2023, time::Month::December, 12).unwrap(); assert_eq!(expected, real); @@ -108,8 +108,4 @@ fn test_wikipedia_date_parse() { parse_wikipedia_date("October 2021").unwrap(), time::Date::from_calendar_date(2021, time::Month::October, 1).unwrap() ); - assert_eq!( - parse_wikipedia_date("october 2021").unwrap(), - time::Date::from_calendar_date(2021, time::Month::October, 1).unwrap() - ); }