remove compile time postgres dependency, add docker build

This commit is contained in:
neri 2020-07-12 02:26:11 +02:00
parent deb99942d3
commit e110378ba6
8 changed files with 112 additions and 74 deletions

11
Cargo.lock generated
View File

@ -688,6 +688,7 @@ dependencies = [
"futures", "futures",
"log", "log",
"mime", "mime",
"openssl-sys",
"rand", "rand",
"sqlx", "sqlx",
] ]
@ -1454,6 +1455,15 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de"
[[package]]
name = "openssl-src"
version = "111.10.1+1.1.1g"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "375f12316ddf0762f7cf1e2890a0a857954b96851b47b5af7fc06940c9e12f83"
dependencies = [
"cc",
]
[[package]] [[package]]
name = "openssl-sys" name = "openssl-sys"
version = "0.9.58" version = "0.9.58"
@ -1463,6 +1473,7 @@ dependencies = [
"autocfg", "autocfg",
"cc", "cc",
"libc", "libc",
"openssl-src",
"pkg-config", "pkg-config",
"vcpkg", "vcpkg",
] ]

View File

@ -19,3 +19,7 @@ futures = "0.3.5"
mime = "0.3.16" mime = "0.3.16"
rand = "0.7.3" rand = "0.7.3"
chrono = "0.4.13" chrono = "0.4.13"
openssl-sys = "*"
[features]
vendored = ["openssl-sys/vendored"]

View File

@ -1,30 +1,31 @@
FROM postgres as builder FROM ekidd/rust-musl-builder as build
ENV POSTGRES_USER "datatrash" USER rust
ENV POSTGRES_PASSWORD "secure" WORKDIR /home/rust/src/
RUN USER=rust cargo new datatrash
RUN apt-get update WORKDIR /home/rust/src/datatrash
RUN apt-get install --yes curl build-essential COPY --chown=rust Cargo.toml Cargo.lock ./
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o rustup-install.sh RUN cargo build --release --features vendored
RUN sh rustup-install.sh -y
ENV PATH="/root/.cargo/bin:${PATH}"
RUN rustup target add x86_64-unknown-linux-musl
ENV USER rust COPY --chown=rust src ./src
WORKDIR / COPY --chown=rust static ./static
RUN cargo new --bin datatrash COPY --chown=rust template ./template
WORKDIR /datatrash COPY --chown=rust init-db.sql ./init-db.sql
COPY Cargo.lock Cargo.lock RUN touch src/main.rs
COPY Cargo.toml Cargo.toml RUN cargo install --path . --features vendored
RUN cargo build --release --target=x86_64-unknown-linux-musl --features vendored
RUN rm src/*.rs
COPY src src FROM alpine
ENV DATABASE_URL "postgresql://datatrash:secure@localhost"
RUN rm target/release/deps/datatrash*
RUN cargo build --release --target=x86_64-unknown-linux-musl --features vendored
RUN strip target/release/horrible
FROM SCRATCH ENV DATABASE_URL "postresql://localhost"
COPY --from=builder /datatrash/target/release/datatrash /datatrash ENV SERVER_URL "http://localhost:8000"
ENTRYPOINT ["/datatrash"] ENV FILES_DIR "./files"
ENV UPLOAD_MAX_BYTES "8388608"
ENV BIND_ADDRESS "0.0.0.0:8000"
ENV RUST_BACKTRACE "1"
COPY --from=build /home/rust/.cargo/bin/datatrash .
COPY static ./static
RUN mkdir ./files
EXPOSE 8000
ENTRYPOINT ["./datatrash"]

View File

@ -6,16 +6,16 @@ A file and text uploading service with configurable time limit
## compiling ## compiling
Compiling is a little strange. ```sh
The SQL-statements are checked for correctness at compile-time, unfortunately this means that the docker build -t datatrash .
database needs to be running at compile-time too. docker cp datatrash:/home/rust/.cargo/bin/datatrash datatrash
```
To get set up: or, to just run it in docker
- Start a postgresql somewhere ```sh
- Set its connection url in the `.env` file docker-compose up -d --build
- Run the `init-db.sql` script in the database (`cat init-db.sql | psql`) ```
- Build the project `cargo build --release`
## running & config ## running & config
@ -27,6 +27,4 @@ To get set up:
| UPLOAD_MAX_BYTES | 8388608 (8MiB) | | UPLOAD_MAX_BYTES | 8388608 (8MiB) |
| BIND_ADDRESS | 0.0.0.0:8000 | | BIND_ADDRESS | 0.0.0.0:8000 |
Other things are not configurable yet. The maximum filename length is 255
- The maximum filename length is 255

15
docker-compose.yml Normal file
View File

@ -0,0 +1,15 @@
version: "3.3"
services:
datatrash:
build: .
environment:
DATABASE_URL: 'postgresql://admin:secure@postgres'
ports:
- '8000:8000'
postgres:
image: postgres
environment:
POSTGRES_USER: admin
POSTGRES_PASSWORD: secure

View File

@ -1,26 +1,26 @@
use async_std::{fs, path::PathBuf, sync::Receiver, task}; use async_std::{fs, path::PathBuf, sync::Receiver, task};
use chrono::{prelude::*, Duration}; use chrono::{prelude::*, Duration};
use futures::future::FutureExt; use futures::future::FutureExt;
use sqlx::postgres::PgPool; use sqlx::{postgres::PgPool, Cursor, Row};
pub(crate) async fn delete_old_files(receiver: Receiver<()>, db: PgPool, files_dir: PathBuf) { pub(crate) async fn delete_old_files(receiver: Receiver<()>, db: PgPool, files_dir: PathBuf) {
loop { loop {
wait_for_file_expiry(&receiver, &db).await; wait_for_file_expiry(&receiver, &db).await;
let now = Local::now().naive_local(); let mut cursor = sqlx::query("SELECT file_id FROM files WHERE files.valid_till < $1")
let expired_files = .bind(Local::now().naive_local())
sqlx::query!("SELECT file_id FROM files WHERE files.valid_till < $1", now) .fetch(&db);
.fetch_all(&db)
.await while let Some(row) = cursor.next().await.expect("could not load expired files") {
.unwrap(); let file_id: String = row.get("file_id");
for expired_file in expired_files {
let mut path = files_dir.clone(); let mut path = files_dir.clone();
path.push(&expired_file.file_id); path.push(&file_id);
if path.exists().await { if path.exists().await {
log::info!("delete file {}", expired_file.file_id); log::info!("delete file {}", file_id);
fs::remove_file(&path).await.expect("could not delete file"); fs::remove_file(&path).await.expect("could not delete file");
} }
} }
sqlx::query!("DELETE FROM files WHERE valid_till < $1", now) sqlx::query("DELETE FROM files WHERE valid_till < $1")
.bind(Local::now().naive_local())
.execute(&db) .execute(&db)
.await .await
.expect("could not delete expired files from database"); .expect("could not delete expired files from database");
@ -28,12 +28,15 @@ pub(crate) async fn delete_old_files(receiver: Receiver<()>, db: PgPool, files_d
} }
async fn wait_for_file_expiry(receiver: &Receiver<()>, db: &PgPool) { async fn wait_for_file_expiry(receiver: &Receiver<()>, db: &PgPool) {
let row = sqlx::query!("SELECT MIN(valid_till) as min from files") let mut cursor = sqlx::query("SELECT MIN(valid_till) as min from files").fetch(db);
.fetch_one(db) let row = cursor
.next()
.await .await
.expect("could not fetch expiring file from database"); .expect("could not fetch expiring files from database")
let next_timeout = match row.min { .expect("postgres min did not return any row");
Some(min) => min.signed_duration_since(Local::now().naive_local()), let valid_till: Option<NaiveDateTime> = row.get("min");
let next_timeout = match valid_till {
Some(valid_till) => valid_till.signed_duration_since(Local::now().naive_local()),
None => Duration::days(1), None => Duration::days(1),
}; };
let positive_timeout = next_timeout let positive_timeout = next_timeout

View File

@ -13,7 +13,10 @@ use actix_web::{
}; };
use async_std::{fs, path::PathBuf, sync::Sender, task}; use async_std::{fs, path::PathBuf, sync::Sender, task};
use file_kind::FileKind; use file_kind::FileKind;
use sqlx::postgres::PgPool; use sqlx::{
postgres::{PgPool, PgRow},
Cursor, Row,
};
use std::env; use std::env;
const INDEX_HTML: &str = include_str!("../template/index.html"); const INDEX_HTML: &str = include_str!("../template/index.html");
@ -49,16 +52,14 @@ async fn upload(
} }
}; };
sqlx::query!( sqlx::query("INSERT INTO Files (file_id, file_name, valid_till, kind) VALUES ($1, $2, $3, $4)")
"INSERT INTO Files (file_id, file_name, valid_till, kind) VALUES ($1, $2, $3, $4)", .bind(&file_id)
file_id, .bind(original_name.unwrap_or_else(|| file_id.clone()))
original_name.unwrap_or_else(|| file_id.clone()), .bind(valid_till.naive_local())
valid_till.naive_local(), .bind(kind.to_string())
kind.to_string() .execute(db.as_ref())
) .await
.execute(db.as_ref()) .map_err(|_| error::ErrorInternalServerError("could not insert file into database"))?;
.await
.map_err(|_| error::ErrorInternalServerError("could not insert file into database"))?;
log::info!( log::info!(
"create new file {} (valid_till: {}, kind: {})", "create new file {} (valid_till: {}, kind: {})",
@ -89,17 +90,21 @@ async fn download(
db: web::Data<PgPool>, db: web::Data<PgPool>,
config: web::Data<Config>, config: web::Data<Config>,
) -> Result<HttpResponse, Error> { ) -> Result<HttpResponse, Error> {
let row = sqlx::query!( let mut cursor = sqlx::query("SELECT file_id, file_name, kind from files WHERE file_id = $1")
"SELECT file_id, file_name, kind from files WHERE file_id = $1", .bind(id.as_ref())
*id .fetch(db.as_ref());
) let row: PgRow = cursor
.fetch_one(db.as_ref()) .next()
.await .await
.map_err(|_| error::ErrorNotFound("could not find file"))?; .map_err(|_| error::ErrorInternalServerError("could not run select statement"))?
.ok_or_else(|| error::ErrorNotFound("could not find file"))?;
let file_id: String = row.get("file_id");
let file_name: String = row.get("file_name");
let kind: String = row.get("kind");
let mut path = config.files_dir.clone(); let mut path = config.files_dir.clone();
path.push(&row.file_id); path.push(&file_id);
if row.kind == FileKind::TEXT.to_string() { if kind == FileKind::TEXT.to_string() {
let content = fs::read_to_string(path).await?; let content = fs::read_to_string(path).await?;
let view_html = VIEW_HTML.replace("{text}", &content); let view_html = VIEW_HTML.replace("{text}", &content);
let response = HttpResponse::Ok().content_type("text/html").body(view_html); let response = HttpResponse::Ok().content_type("text/html").body(view_html);
@ -107,7 +112,7 @@ async fn download(
} else { } else {
let file = NamedFile::open(path)?.set_content_disposition(ContentDisposition { let file = NamedFile::open(path)?.set_content_disposition(ContentDisposition {
disposition: DispositionType::Attachment, disposition: DispositionType::Attachment,
parameters: vec![DispositionParam::Filename(row.file_name)], parameters: vec![DispositionParam::Filename(file_name)],
}); });
file.into_response(&req) file.into_response(&req)
} }
@ -120,7 +125,7 @@ async fn setup_db() -> PgPool {
.await .await
.expect("could not create db pool"); .expect("could not create db pool");
sqlx::query_file!("./init-db.sql") sqlx::query(include_str!("../init-db.sql"))
.execute(&pool) .execute(&pool)
.await .await
.expect("could not create table Files"); .expect("could not create table Files");

View File

@ -20,6 +20,7 @@
<br /> <br />
Gültig für Gültig für
<select name="validity_secs"> <select name="validity_secs">
<option value="10">10 sekunden</option>
<option value="1800">30 minuten</option> <option value="1800">30 minuten</option>
<option value="3600">60 minuten</option> <option value="3600">60 minuten</option>
<option value="43200">12 stunden</option> <option value="43200">12 stunden</option>