manuth/PortValhalla
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add scripts for installing firefox sync server

Browse source
This commit is contained in:
Manuel Thalmann 2025-03-10 23:35:29 +01:00
parent 9857514544
commit 56fc7da02c
10 changed files with 339 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
lib/modules/programs/docker.nix
View file

@ -27,6 +27,7 @@ in
services = {
anki-sync.enable = mkEnableOption "Anki Sync server";
drone.enable = mkEnableOption "drone server";
firefox-sync.enable = mkEnableOption "Firefox Sync server";
forgejo.enable = mkEnableOption "Forgejo server";
gotify.enable = mkEnableOption "Gotify server";
jellyfin.enable = mkEnableOption "Jellyfin media server";

1
profiles/machines/manuel/server.nix
View file

@ -108,6 +108,7 @@ in {
services = {
anki-sync.enable = true;
drone.enable = true;
firefox-sync.enable = true;
forgejo.enable = true;
gotify.enable = true;
jellyfin.enable = true;

2
scripts/Common/Software/docker/services/firefox-sync/.dockerignore Normal file
View file

@ -0,0 +1,2 @@
*
!*.patch

28
scripts/Common/Software/docker/services/firefox-sync/Dockerfile Normal file
View file

@ -0,0 +1,28 @@
FROM rust AS build
ARG SYNC_STORAGE_VERSION=0.18.2
RUN git clone https://github.com/mozilla-services/syncstorage-rs -b ${SYNC_STORAGE_VERSION} /app
WORKDIR /app
COPY ./public-url.patch .
RUN \
apt-get update \
&& apt-get install -y libpython3-dev \
&& git apply public-url.patch \
&& cargo install --path ./syncserver --features mysql --locked \
&& cargo install diesel_cli --no-default-features --features mysql --locked \
&& cargo clean \
&& apt-get remove -y libpython3-dev \
&& rm -rf /var/lib/apt/lists \
&& bash -O extglob -c 'rm -rf /usr/local/cargo/!(bin)' \
&& bash -O extglob -c 'rm -rf /usr/local/cargo/bin/!(diesel|syncserver)'
FROM python:3.11 AS sync
COPY --from=build /usr/local/cargo/bin/syncserver /usr/local/bin
COPY --from=build /app/requirements.txt .
RUN pip install -r requirements.txt
CMD [ "/usr/local/bin/syncserver" ]
FROM mariadb AS db
RUN mkdir -p /app/tokenserver-db
COPY --from=build /app/tokenserver-db/migrations /app/tokenserver-db/migrations
COPY --from=build /usr/local/cargo/bin/diesel /usr/local/bin

37
scripts/Common/Software/docker/services/firefox-sync/docker-compose.base.yml Normal file
View file

@ -0,0 +1,37 @@
services:
sync-server:
build:
context: .
target: sync
restart: unless-stopped
environment:
RUST_LOG: warn
SYNC_HUMAN_LOGS: 1
SYNC_HOST: "0.0.0.0"
SYNC_PORT: 80
SYNC_SYNCSTORAGE__ENABLED: "true"
SYNC_SYNCSTORAGE__ENABLE_QUOTA: 0
SYNC_TOKENSERVER__ENABLED: "true"
SYNC_TOKENSERVER__RUN_MIGRATIONS: "true"
SYNC_TOKENSERVER__FXA_EMAIL_DOMAIN: api.accounts.firefox.com
SYNC_TOKENSERVER__FXA_OAUTH_SERVER_URL: https://oauth.accounts.firefox.com
SYNC_TOKENSERVER__FXA_BROWSERID_AUDIENCE: https://token.services.mozilla.com
SYNC_TOKENSERVER__FXA_BROWSERID_ISSUER: https://api.accounts.firefox.com
SYNC_TOKENSERVER__FXA_BROWSERID_SERVER_URL: https://verifier.accounts.firefox.com/v2
sync-db:
image: mariadb
restart: unless-stopped
environment:
MARIADB_RANDOM_ROOT_PASSWORD: "yes"
volumes:
- ./data/sync:/var/lib/mysql
token-db:
build:
context: .
target: db
restart: unless-stopped
environment:
MARIADB_RANDOM_ROOT_PASSWORD: "yes"
volumes:
- ./data/token:/var/lib/mysql
- ./init:/docker-entrypoint-initdb.d

21
scripts/Common/Software/docker/services/firefox-sync/docker-compose.overrides.yml Normal file
View file

@ -0,0 +1,21 @@
services:
sync-server:
environment:
SYNC_PUBLIC_URL: https://example.com
SYNC_MASTER_SECRET: secret
SYNC_SYNCSTORAGE__DATABASE_URL: mysql://sync:password@sync-db/SyncStorage
SYNC_TOKENSERVER__DATABASE_URL: mysql://token:password@token-db/TokenServer
SYNC_TOKENSERVER__FXA_METRICS_HASH_SECRET: secret
ports:
- 127.0.0.1:1337:80
sync-db:
environment:
MARIADB_USER: sync
MARIADB_PASSWORD: password
MARIADB_DATABASE: SyncStorage
token-db:
environment:
SYNC_PUBLIC_URL: https://example.com
MARIADB_USER: token
MARIADB_PASSWORD: password
MARIADB_DATABASE: TokenServer

2
scripts/Common/Software/docker/services/firefox-sync/init/z1-migrate.sh Normal file
View file

@ -0,0 +1,2 @@
#!/bin/bash
diesel --database-url "mysql://${MARIADB_USER}:${MARIADB_PASSWORD}@localhost/${MARIADB_DATABASE}" migration --migration-dir /app/tokenserver-db/migrations run

10
scripts/Common/Software/docker/services/firefox-sync/init/z2-add-service.sh Normal file
View file

@ -0,0 +1,10 @@
mariadb -u$MARIADB_USER -p$MARIADB_PASSWORD -D $MARIADB_DATABASE <<EOF
INSERT INTO services
(service, pattern)
SELECT 'sync-1.5', '{node}/1.5/{uid}'
WHERE NOT EXISTS (SELECT 1 FROM services);
INSERT INTO nodes
(\`service\`, node, capacity, available, current_load, downed, backoff)
VALUES (LAST_INSERT_ID(), '${SYNC_PUBLIC_URL}', 1, 1, 0, 0, 0)
EOF

64
scripts/Common/Software/docker/services/firefox-sync/main.fish Normal file
View file

@ -0,0 +1,64 @@
#!/bin/env fish
begin
set -l domain sync
set -l dir (status dirname)
set -l source "$dir/docker-compose.overrides.yml"
source "$dir/../service.fish"
function installSW -V dir -V source
set -l root (getServiceRoot $argv)
initializeServiceInstallation $argv
sudo cp \
"$dir/"{docker-compose.{base,overrides}.yml,.dockerignore,Dockerfile,public-url.patch} \
"$root"
sudo cp -r "$dir/init" "$root"
installDockerService $argv
end
function configureSW -V dir -V domain
set -l overrides (getServiceOverrides $argv)
set -l envKey ".services.sync-server.environment"
set -l url https://$domain.nuth.ch/firefox
set databases sync SYNCSTORAGE \
token TOKENSERVER
for variable in masterSecret metricsSecret syncPW tokenPW
set $variable (nix-shell -p keepassxc --run "keepassxc-cli generate --length 64")
end
sudo SECRET="$masterSecret" yq -i "$envKey.SYNC_MASTER_SECRET = env(SECRET)" "$overrides"
sudo SECRET="$metricsSecret" yq -i "$envKey.SYNC_TOKENSERVER__FXA_METRICS_HASH_SECRET = env(SECRET)" "$overrides"
sudo URL=$url yq -i "$envKey.SYNC_PUBLIC_URL = env(URL)" "$overrides"
sudo URL=$url yq -i ".services.token-db.environment.SYNC_PUBLIC_URL = env(URL)" "$overrides"
configureDockerService $argv
for i in $(seq 1 2 (count $databases))
set -l prefix $databases[$i]
set -l section $databases[(math "$i + 1")]
set -l pwVar "$prefix""PW"
set -l pw $$pwVar
set -l dbEnv ".services.$prefix-db.environment"
set -l db (yq "$dbEnv.MARIADB_DATABASE" "$overrides")
set -l user (yq "$dbEnv.MARIADB_USER" "$overrides")
sudo PW=$pw yq -i "$dbEnv.MARIADB_PASSWORD = env(PW)" "$overrides"
sudo CONNECTION="mysql://$user:$pw@$prefix-db/$db" yq -i ".services.sync-server.environment.SYNC_"$section"__DATABASE_URL = env(CONNECTION)" "$overrides"
end
end
function getServiceServers -V domain
printf "$domain" ""
end
function getServiceLocations -V domain
printf "%s\0" "sync-server" /firefox/ "Firefox Sync"
end
function getServiceLocationConfig -a domain s location -V service -V flood
getServiceDefaultProxy $argv --path "/"
end
runInstaller --force $argv
end

173
scripts/Common/Software/docker/services/firefox-sync/public-url.patch Normal file
View file

@ -0,0 +1,173 @@
diff --git a/syncserver-settings/src/lib.rs b/syncserver-settings/src/lib.rs
index 6732ffc..1c9ef83 100644
--- a/syncserver-settings/src/lib.rs
+++ b/syncserver-settings/src/lib.rs
@@ -18,6 +18,7 @@ static PREFIX: &str = "sync";
pub struct Settings {
pub port: u16,
pub host: String,
+ pub public_url: Option<String>,
/// Keep-alive header value (seconds)
pub actix_keep_alive: Option<u32>,
/// The master secret, from which are derived
@@ -182,6 +183,7 @@ impl Default for Settings {
Settings {
port: 8000,
host: "127.0.0.1".to_string(),
+ public_url: None,
actix_keep_alive: None,
master_secret: Secrets::default(),
statsd_host: Some("localhost".to_owned()),
diff --git a/syncserver/src/server/mod.rs b/syncserver/src/server/mod.rs
index f1e0a18..d686693 100644
--- a/syncserver/src/server/mod.rs
+++ b/syncserver/src/server/mod.rs
@@ -80,7 +80,7 @@ pub struct Server;
#[macro_export]
macro_rules! build_app {
- ($syncstorage_state: expr, $tokenserver_state: expr, $secrets: expr, $limits: expr, $cors: expr, $metrics: expr) => {
+ ($public_url: expr, $syncstorage_state: expr, $tokenserver_state: expr, $secrets: expr, $limits: expr, $cors: expr, $metrics: expr) => {
App::new()
.configure(|cfg| {
cfg.app_data(Data::new($syncstorage_state));
@@ -90,6 +90,7 @@ macro_rules! build_app {
cfg.app_data(Data::new(state));
}
})
+ .app_data(Data::new($public_url))
.app_data(Data::new($secrets))
// Middleware is applied LIFO
// These will wrap all outbound responses with matching status codes.
@@ -336,6 +337,7 @@ impl Server {
};
build_app!(
+ settings.public_url.clone(),
syncstorage_state,
tokenserver_state.clone(),
Arc::clone(&secrets),
diff --git a/syncserver/src/server/test.rs b/syncserver/src/server/test.rs
index 8690526..7efbe98 100644
--- a/syncserver/src/server/test.rs
+++ b/syncserver/src/server/test.rs
@@ -127,6 +127,7 @@ macro_rules! init_app {
let state = get_test_state(&$settings).await;
let metrics = state.metrics.clone();
test::init_service(build_app!(
+ $settings.public_url.clone(),
state,
None::<tokenserver::ServerState>,
Arc::clone(&SECRETS),
@@ -248,6 +249,7 @@ where
let state = get_test_state(&settings).await;
let metrics = state.metrics.clone();
let app = test::init_service(build_app!(
+ settings.public_url.clone(),
state,
None::<tokenserver::ServerState>,
Arc::clone(&SECRETS),
@@ -292,6 +294,7 @@ async fn test_endpoint_with_body(
let state = get_test_state(&settings).await;
let metrics = state.metrics.clone();
let app = test::init_service(build_app!(
+ settings.public_url.clone(),
state,
None::<tokenserver::ServerState>,
Arc::clone(&SECRETS),
diff --git a/syncserver/src/web/auth.rs b/syncserver/src/web/auth.rs
index 9153d38..ca1d123 100644
--- a/syncserver/src/web/auth.rs
+++ b/syncserver/src/web/auth.rs
@@ -170,6 +170,7 @@ impl HawkPayload {
pub fn extrude(
header: &str,
method: &str,
+ public_url: &Option<String>,
secrets: &Secrets,
ci: &ConnectionInfo,
uri: &Uri,
@@ -190,6 +191,15 @@ impl HawkPayload {
} else {
80
};
+
+ let prefix = match &public_url {
+ None => "".to_owned(),
+ Some(url) => match url.parse::<Uri>() {
+ Err(_) => "".to_owned(),
+ Ok(uri) => uri.path().to_owned()
+ }
+ };
+
let path = uri.path_and_query().ok_or(HawkErrorKind::MissingPath)?;
let expiry = if path.path().ends_with("/info/collections") {
0
@@ -197,7 +207,7 @@ impl HawkPayload {
Utc::now().timestamp() as u64
};
- HawkPayload::new(header, method, path.as_str(), host, port, secrets, expiry)
+ HawkPayload::new(header, method, (prefix + path.as_str()).as_str(), host, port, secrets, expiry)
}
}
diff --git a/syncserver/src/web/extractors.rs b/syncserver/src/web/extractors.rs
index 4bd617b..b9aee1c 100644
--- a/syncserver/src/web/extractors.rs
+++ b/syncserver/src/web/extractors.rs
@@ -1056,6 +1056,7 @@ impl HawkIdentifier {
method: &str,
uri: &Uri,
ci: &ConnectionInfo,
+ public_url: &Option<String>,
secrets: &Secrets,
) -> Result<Self, Error>
where
@@ -1072,6 +1073,7 @@ impl HawkIdentifier {
.to_str()
.map_err(|e| -> ApiError { HawkErrorKind::Header(e).into() })?;
let identifier = Self::generate(
+ public_url,
secrets,
method,
auth_header,
@@ -1084,6 +1086,7 @@ impl HawkIdentifier {
}
pub fn generate(
+ public_url: &Option<String>,
secrets: &Secrets,
method: &str,
header: &str,
@@ -1091,7 +1094,7 @@ impl HawkIdentifier {
uri: &Uri,
exts: &mut Extensions,
) -> Result<Self, Error> {
- let payload = HawkPayload::extrude(header, method, secrets, connection_info, uri)?;
+ let payload = HawkPayload::extrude(header, method, &public_url, secrets, connection_info, uri)?;
let puid = Self::uid_from_path(uri)?;
if payload.user_id != puid {
warn!("⚠️ Hawk UID not in URI: {:?} {:?}", payload.user_id, uri);
@@ -1145,6 +1148,12 @@ impl FromRequest for HawkIdentifier {
// NOTE: `connection_info()` will get a mutable reference lock on `extensions()`
let connection_info = req.connection_info().clone();
let method = req.method().clone();
+
+ let public_url: &Option<String> = match &req.app_data::<Data<Option<String>>>() {
+ Some(data) => data,
+ None => &None
+ };
+
// Tried collapsing this to a `.or_else` and hit problems with the return resolving
// to an appropriate error state. Can't use `?` since the function does not return a result.
let secrets = match req.app_data::<Data<Arc<Secrets>>>() {
@@ -1155,7 +1164,7 @@ impl FromRequest for HawkIdentifier {
}
};
- let result = Self::extrude(&req, method.as_str(), uri, &connection_info, secrets);
+ let result = Self::extrude(&req, method.as_str(), uri, &connection_info, &public_url, secrets);
if let Ok(ref hawk_id) = result {
// Store the origin of the token as an extra to be included when emitting a Sentry error