Add scripts for installing firefox sync server

lib/modules/programs/docker.nix

@@ -27,6 +27,7 @@ in
             services = {
               anki-sync.enable = mkEnableOption "Anki Sync server";
               drone.enable = mkEnableOption "drone server";
+              firefox-sync.enable = mkEnableOption "Firefox Sync server";
               forgejo.enable = mkEnableOption "Forgejo server";
               gotify.enable = mkEnableOption "Gotify server";
               jellyfin.enable = mkEnableOption "Jellyfin media server";

profiles/machines/manuel/server.nix

@@ -108,6 +108,7 @@ in {
           services = {
             anki-sync.enable = true;
             drone.enable = true;
+            firefox-sync.enable = true;
             forgejo.enable = true;
             gotify.enable = true;
             jellyfin.enable = true;

scripts/Common/Software/docker/services/firefox-sync/.dockerignore

@@ -0,0 +1,2 @@
+*
+!*.patch

scripts/Common/Software/docker/services/firefox-sync/Dockerfile

@@ -0,0 +1,28 @@
+FROM rust AS build
+ARG SYNC_STORAGE_VERSION=0.18.2
+RUN git clone https://github.com/mozilla-services/syncstorage-rs -b ${SYNC_STORAGE_VERSION} /app
+WORKDIR /app
+COPY ./public-url.patch .
+
+RUN \
+    apt-get update \
+    && apt-get install -y libpython3-dev \
+    && git apply public-url.patch \
+    && cargo install --path ./syncserver --features mysql --locked \
+    && cargo install diesel_cli --no-default-features --features mysql --locked \
+    && cargo clean \
+    && apt-get remove -y libpython3-dev \
+    && rm -rf /var/lib/apt/lists \
+    && bash -O extglob -c 'rm -rf /usr/local/cargo/!(bin)' \
+    && bash -O extglob -c 'rm -rf /usr/local/cargo/bin/!(diesel|syncserver)'
+
+FROM python:3.11 AS sync
+COPY --from=build /usr/local/cargo/bin/syncserver /usr/local/bin
+COPY --from=build /app/requirements.txt .
+RUN pip install -r requirements.txt
+CMD [ "/usr/local/bin/syncserver" ]
+
+FROM mariadb AS db
+RUN mkdir -p /app/tokenserver-db
+COPY --from=build /app/tokenserver-db/migrations /app/tokenserver-db/migrations
+COPY --from=build /usr/local/cargo/bin/diesel /usr/local/bin

scripts/Common/Software/docker/services/firefox-sync/docker-compose.base.yml

@@ -0,0 +1,37 @@
+services:
+  sync-server:
+    build:
+      context: .
+      target: sync
+    restart: unless-stopped
+    environment:
+      RUST_LOG: warn
+      SYNC_HUMAN_LOGS: 1
+      SYNC_HOST: "0.0.0.0"
+      SYNC_PORT: 80
+      SYNC_SYNCSTORAGE__ENABLED: "true"
+      SYNC_SYNCSTORAGE__ENABLE_QUOTA: 0
+      SYNC_TOKENSERVER__ENABLED: "true"
+      SYNC_TOKENSERVER__RUN_MIGRATIONS: "true"
+      SYNC_TOKENSERVER__FXA_EMAIL_DOMAIN: api.accounts.firefox.com
+      SYNC_TOKENSERVER__FXA_OAUTH_SERVER_URL: https://oauth.accounts.firefox.com
+      SYNC_TOKENSERVER__FXA_BROWSERID_AUDIENCE: https://token.services.mozilla.com
+      SYNC_TOKENSERVER__FXA_BROWSERID_ISSUER: https://api.accounts.firefox.com
+      SYNC_TOKENSERVER__FXA_BROWSERID_SERVER_URL: https://verifier.accounts.firefox.com/v2
+  sync-db:
+    image: mariadb
+    restart: unless-stopped
+    environment:
+      MARIADB_RANDOM_ROOT_PASSWORD: "yes"
+    volumes:
+      - ./data/sync:/var/lib/mysql
+  token-db:
+    build:
+      context: .
+      target: db
+    restart: unless-stopped
+    environment:
+      MARIADB_RANDOM_ROOT_PASSWORD: "yes"
+    volumes:
+      - ./data/token:/var/lib/mysql
+      - ./init:/docker-entrypoint-initdb.d

scripts/Common/Software/docker/services/firefox-sync/docker-compose.overrides.yml

@@ -0,0 +1,21 @@
+services:
+  sync-server:
+    environment:
+      SYNC_PUBLIC_URL: https://example.com
+      SYNC_MASTER_SECRET: secret
+      SYNC_SYNCSTORAGE__DATABASE_URL: mysql://sync:password@sync-db/SyncStorage
+      SYNC_TOKENSERVER__DATABASE_URL: mysql://token:password@token-db/TokenServer
+      SYNC_TOKENSERVER__FXA_METRICS_HASH_SECRET: secret
+    ports:
+      - 127.0.0.1:1337:80
+  sync-db:
+    environment:
+      MARIADB_USER: sync
+      MARIADB_PASSWORD: password
+      MARIADB_DATABASE: SyncStorage
+  token-db:
+    environment:
+      SYNC_PUBLIC_URL: https://example.com
+      MARIADB_USER: token
+      MARIADB_PASSWORD: password
+      MARIADB_DATABASE: TokenServer

scripts/Common/Software/docker/services/firefox-sync/init/z1-migrate.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+diesel --database-url "mysql://${MARIADB_USER}:${MARIADB_PASSWORD}@localhost/${MARIADB_DATABASE}" migration --migration-dir /app/tokenserver-db/migrations run

scripts/Common/Software/docker/services/firefox-sync/init/z2-add-service.sh

@@ -0,0 +1,10 @@
+mariadb -u$MARIADB_USER -p$MARIADB_PASSWORD -D $MARIADB_DATABASE <<EOF
+INSERT INTO services
+(service, pattern)
+SELECT 'sync-1.5', '{node}/1.5/{uid}'
+WHERE NOT EXISTS (SELECT 1 FROM services);
+
+INSERT INTO nodes
+(\`service\`, node, capacity, available, current_load, downed, backoff)
+VALUES (LAST_INSERT_ID(), '${SYNC_PUBLIC_URL}', 1, 1, 0, 0, 0)
+EOF

scripts/Common/Software/docker/services/firefox-sync/main.fish

@@ -0,0 +1,64 @@
+#!/bin/env fish
+begin
+    set -l domain sync
+    set -l dir (status dirname)
+    set -l source "$dir/docker-compose.overrides.yml"
+    source "$dir/../service.fish"
+
+    function installSW -V dir -V source
+        set -l root (getServiceRoot $argv)
+        initializeServiceInstallation $argv
+
+        sudo cp \
+            "$dir/"{docker-compose.{base,overrides}.yml,.dockerignore,Dockerfile,public-url.patch} \
+            "$root"
+
+	sudo cp -r "$dir/init" "$root"
+        installDockerService $argv
+    end
+
+    function configureSW -V dir -V domain
+        set -l overrides (getServiceOverrides $argv)
+        set -l envKey ".services.sync-server.environment"
+        set -l url https://$domain.nuth.ch/firefox
+
+        set databases sync SYNCSTORAGE \
+            token TOKENSERVER
+
+        for variable in masterSecret metricsSecret syncPW tokenPW
+            set $variable (nix-shell -p keepassxc --run "keepassxc-cli generate --length 64")
+        end
+
+        sudo SECRET="$masterSecret" yq -i "$envKey.SYNC_MASTER_SECRET = env(SECRET)" "$overrides"
+        sudo SECRET="$metricsSecret" yq -i "$envKey.SYNC_TOKENSERVER__FXA_METRICS_HASH_SECRET = env(SECRET)" "$overrides"
+        sudo URL=$url yq -i "$envKey.SYNC_PUBLIC_URL = env(URL)" "$overrides"
+        sudo URL=$url yq -i ".services.token-db.environment.SYNC_PUBLIC_URL = env(URL)" "$overrides"
+        configureDockerService $argv
+
+        for i in $(seq 1 2 (count $databases))
+            set -l prefix $databases[$i]
+            set -l section $databases[(math "$i + 1")]
+            set -l pwVar "$prefix""PW"
+            set -l pw $$pwVar
+            set -l dbEnv ".services.$prefix-db.environment"
+            set -l db (yq "$dbEnv.MARIADB_DATABASE" "$overrides")
+            set -l user (yq "$dbEnv.MARIADB_USER" "$overrides")
+            sudo PW=$pw yq -i "$dbEnv.MARIADB_PASSWORD = env(PW)" "$overrides"
+            sudo CONNECTION="mysql://$user:$pw@$prefix-db/$db" yq -i ".services.sync-server.environment.SYNC_"$section"__DATABASE_URL = env(CONNECTION)" "$overrides"
+        end
+    end
+
+    function getServiceServers -V domain
+        printf "$domain" ""
+    end
+
+    function getServiceLocations -V domain
+        printf "%s\0" "sync-server" /firefox/ "Firefox Sync"
+    end
+
+    function getServiceLocationConfig -a domain s location -V service -V flood
+        getServiceDefaultProxy $argv --path "/"
+    end
+
+    runInstaller --force $argv
+end

scripts/Common/Software/docker/services/firefox-sync/public-url.patch

@@ -0,0 +1,173 @@
+diff --git a/syncserver-settings/src/lib.rs b/syncserver-settings/src/lib.rs
+index 6732ffc..1c9ef83 100644
+--- a/syncserver-settings/src/lib.rs
++++ b/syncserver-settings/src/lib.rs
+@@ -18,6 +18,7 @@ static PREFIX: &str = "sync";
+ pub struct Settings {
+     pub port: u16,
+     pub host: String,
++    pub public_url: Option<String>,
+     /// Keep-alive header value (seconds)
+     pub actix_keep_alive: Option<u32>,
+     /// The master secret, from which are derived
+@@ -182,6 +183,7 @@ impl Default for Settings {
+         Settings {
+             port: 8000,
+             host: "127.0.0.1".to_string(),
++            public_url: None,
+             actix_keep_alive: None,
+             master_secret: Secrets::default(),
+             statsd_host: Some("localhost".to_owned()),
+diff --git a/syncserver/src/server/mod.rs b/syncserver/src/server/mod.rs
+index f1e0a18..d686693 100644
+--- a/syncserver/src/server/mod.rs
++++ b/syncserver/src/server/mod.rs
+@@ -80,7 +80,7 @@ pub struct Server;
+ 
+ #[macro_export]
+ macro_rules! build_app {
+-    ($syncstorage_state: expr, $tokenserver_state: expr, $secrets: expr, $limits: expr, $cors: expr, $metrics: expr) => {
++    ($public_url: expr, $syncstorage_state: expr, $tokenserver_state: expr, $secrets: expr, $limits: expr, $cors: expr, $metrics: expr) => {
+         App::new()
+             .configure(|cfg| {
+                 cfg.app_data(Data::new($syncstorage_state));
+@@ -90,6 +90,7 @@ macro_rules! build_app {
+                     cfg.app_data(Data::new(state));
+                 }
+             })
++            .app_data(Data::new($public_url))
+             .app_data(Data::new($secrets))
+             // Middleware is applied LIFO
+             // These will wrap all outbound responses with matching status codes.
+@@ -336,6 +337,7 @@ impl Server {
+             };
+ 
+             build_app!(
++                settings.public_url.clone(),
+                 syncstorage_state,
+                 tokenserver_state.clone(),
+                 Arc::clone(&secrets),
+diff --git a/syncserver/src/server/test.rs b/syncserver/src/server/test.rs
+index 8690526..7efbe98 100644
+--- a/syncserver/src/server/test.rs
++++ b/syncserver/src/server/test.rs
+@@ -127,6 +127,7 @@ macro_rules! init_app {
+             let state = get_test_state(&$settings).await;
+             let metrics = state.metrics.clone();
+             test::init_service(build_app!(
++                $settings.public_url.clone(),
+                 state,
+                 None::<tokenserver::ServerState>,
+                 Arc::clone(&SECRETS),
+@@ -248,6 +249,7 @@ where
+     let state = get_test_state(&settings).await;
+     let metrics = state.metrics.clone();
+     let app = test::init_service(build_app!(
++        settings.public_url.clone(),
+         state,
+         None::<tokenserver::ServerState>,
+         Arc::clone(&SECRETS),
+@@ -292,6 +294,7 @@ async fn test_endpoint_with_body(
+     let state = get_test_state(&settings).await;
+     let metrics = state.metrics.clone();
+     let app = test::init_service(build_app!(
++        settings.public_url.clone(),
+         state,
+         None::<tokenserver::ServerState>,
+         Arc::clone(&SECRETS),
+diff --git a/syncserver/src/web/auth.rs b/syncserver/src/web/auth.rs
+index 9153d38..ca1d123 100644
+--- a/syncserver/src/web/auth.rs
++++ b/syncserver/src/web/auth.rs
+@@ -170,6 +170,7 @@ impl HawkPayload {
+     pub fn extrude(
+         header: &str,
+         method: &str,
++        public_url: &Option<String>,
+         secrets: &Secrets,
+         ci: &ConnectionInfo,
+         uri: &Uri,
+@@ -190,6 +191,15 @@ impl HawkPayload {
+         } else {
+             80
+         };
++
++        let prefix = match &public_url {
++            None => "".to_owned(),
++            Some(url) => match url.parse::<Uri>() {
++                Err(_) => "".to_owned(),
++                Ok(uri) => uri.path().to_owned()
++            }
++        };
++
+         let path = uri.path_and_query().ok_or(HawkErrorKind::MissingPath)?;
+         let expiry = if path.path().ends_with("/info/collections") {
+             0
+@@ -197,7 +207,7 @@ impl HawkPayload {
+             Utc::now().timestamp() as u64
+         };
+ 
+-        HawkPayload::new(header, method, path.as_str(), host, port, secrets, expiry)
++        HawkPayload::new(header, method, (prefix + path.as_str()).as_str(), host, port, secrets, expiry)
+     }
+ }
+ 
+diff --git a/syncserver/src/web/extractors.rs b/syncserver/src/web/extractors.rs
+index 4bd617b..b9aee1c 100644
+--- a/syncserver/src/web/extractors.rs
++++ b/syncserver/src/web/extractors.rs
+@@ -1056,6 +1056,7 @@ impl HawkIdentifier {
+         method: &str,
+         uri: &Uri,
+         ci: &ConnectionInfo,
++        public_url: &Option<String>,
+         secrets: &Secrets,
+     ) -> Result<Self, Error>
+     where
+@@ -1072,6 +1073,7 @@ impl HawkIdentifier {
+             .to_str()
+             .map_err(|e| -> ApiError { HawkErrorKind::Header(e).into() })?;
+         let identifier = Self::generate(
++            public_url,
+             secrets,
+             method,
+             auth_header,
+@@ -1084,6 +1086,7 @@ impl HawkIdentifier {
+     }
+ 
+     pub fn generate(
++        public_url: &Option<String>,
+         secrets: &Secrets,
+         method: &str,
+         header: &str,
+@@ -1091,7 +1094,7 @@ impl HawkIdentifier {
+         uri: &Uri,
+         exts: &mut Extensions,
+     ) -> Result<Self, Error> {
+-        let payload = HawkPayload::extrude(header, method, secrets, connection_info, uri)?;
++        let payload = HawkPayload::extrude(header, method, &public_url, secrets, connection_info, uri)?;
+         let puid = Self::uid_from_path(uri)?;
+         if payload.user_id != puid {
+             warn!("⚠️ Hawk UID not in URI: {:?} {:?}", payload.user_id, uri);
+@@ -1145,6 +1148,12 @@ impl FromRequest for HawkIdentifier {
+         // NOTE: `connection_info()` will get a mutable reference lock on `extensions()`
+         let connection_info = req.connection_info().clone();
+         let method = req.method().clone();
++
++        let public_url: &Option<String> = match &req.app_data::<Data<Option<String>>>() {
++            Some(data) => data,
++            None => &None
++        };
++
+         // Tried collapsing this to a `.or_else` and hit problems with the return resolving
+         // to an appropriate error state. Can't use `?` since the function does not return a result.
+         let secrets = match req.app_data::<Data<Arc<Secrets>>>() {
+@@ -1155,7 +1164,7 @@ impl FromRequest for HawkIdentifier {
+             }
+         };
+ 
+-        let result = Self::extrude(&req, method.as_str(), uri, &connection_info, secrets);
++        let result = Self::extrude(&req, method.as_str(), uri, &connection_info, &public_url, secrets);
+ 
+         if let Ok(ref hawk_id) = result {
+             // Store the origin of the token as an extra to be included when emitting a Sentry error