latexgrambot: inline LaTeX bot for Telegram in Go
Renders LaTeX expressions into PNG pictures, PDF documents and native Telegram rich text messages, entirely locally with TeX Live and poppler. - Chat mode with /image, /pdf, /rich plus per-user preamble and DPI - Inline mode delivering picture, PDF and rich text results - Broad default package set preloaded into a TeX format at image build - MongoDB persistence: settings, users, groups, render statistics - Private S3-compatible bucket with short-lived presigned URLs - OpenTelemetry traces over OTLP - Kubernetes manifests and Argo CD application, Docker Compose, GitHub Actions CI, BSD 3-Clause license
This commit is contained in:
corrado.mulas
145
internal/files/files.go
Normal file
145
internal/files/files.go
Normal file
@@ -0,0 +1,145 @@
|
||||
// Package files stores rendered outputs in a private S3-compatible bucket
|
||||
// and issues short-lived presigned URLs for Telegram inline-query results.
|
||||
package files
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Config points the store at the internal S3-compatible API and the public
|
||||
// API endpoint used only when signing download URLs.
|
||||
type Config struct {
|
||||
Endpoint string
|
||||
Region string
|
||||
Bucket string
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
Prefix string
|
||||
PublicEndpoint string
|
||||
PresignTTL time.Duration
|
||||
HTTPClient aws.HTTPClient
|
||||
}
|
||||
|
||||
// Store keeps content-addressed rendered assets in S3.
|
||||
type Store struct {
|
||||
client *s3.Client
|
||||
presigner *s3.PresignClient
|
||||
bucket string
|
||||
prefix string
|
||||
presignTTL time.Duration
|
||||
}
|
||||
|
||||
// New builds an S3-compatible object store.
|
||||
func New(ctx context.Context, cfg Config) (*Store, error) {
|
||||
awsCfg, err := awsconfig.LoadDefaultConfig(ctx,
|
||||
awsconfig.WithRegion(cfg.Region),
|
||||
awsconfig.WithCredentialsProvider(
|
||||
credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load aws config: %w", err)
|
||||
}
|
||||
if cfg.HTTPClient != nil {
|
||||
awsCfg.HTTPClient = cfg.HTTPClient
|
||||
}
|
||||
client := s3.NewFromConfig(awsCfg, func(options *s3.Options) {
|
||||
options.BaseEndpoint = aws.String(cfg.Endpoint)
|
||||
options.UsePathStyle = true
|
||||
})
|
||||
publicClient := s3.NewFromConfig(awsCfg, func(options *s3.Options) {
|
||||
options.BaseEndpoint = aws.String(cfg.PublicEndpoint)
|
||||
options.UsePathStyle = true
|
||||
})
|
||||
return &Store{
|
||||
client: client,
|
||||
presigner: s3.NewPresignClient(publicClient),
|
||||
bucket: cfg.Bucket,
|
||||
prefix: strings.Trim(cfg.Prefix, "/"),
|
||||
presignTTL: cfg.PresignTTL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Put uploads the data under a fresh UUIDv4 object name and returns a
|
||||
// short-lived presigned GET URL.
|
||||
func (s *Store) Put(ctx context.Context, ext, contentType string, data []byte) (string, error) {
|
||||
key := s.key(uuid.NewString(), ext)
|
||||
_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
Body: bytes.NewReader(data),
|
||||
ContentType: aws.String(contentType),
|
||||
CacheControl: aws.String("private, max-age=86400"),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("put %s object: %w", ext, err)
|
||||
}
|
||||
|
||||
request, err := s.presigner.PresignGetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
}, func(options *s3.PresignOptions) {
|
||||
options.Expires = s.presignTTL
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("presign %s object: %w", ext, err)
|
||||
}
|
||||
return request.URL, nil
|
||||
}
|
||||
|
||||
// DeleteExpired removes objects older than age and returns how many were
|
||||
// deleted. Telegram downloads the file when an inline result is picked and
|
||||
// the presigned link expires after the presign TTL; objects past their
|
||||
// lifetime can no longer be fetched, so they are garbage collected.
|
||||
func (s *Store) DeleteExpired(ctx context.Context, age time.Duration) (int, error) {
|
||||
cutoff := time.Now().Add(-age)
|
||||
prefix := ""
|
||||
if s.prefix != "" {
|
||||
prefix = s.prefix + "/"
|
||||
}
|
||||
deleted := 0
|
||||
var token *string
|
||||
for {
|
||||
list, err := s.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Prefix: aws.String(prefix),
|
||||
ContinuationToken: token,
|
||||
})
|
||||
if err != nil {
|
||||
return deleted, fmt.Errorf("list objects: %w", err)
|
||||
}
|
||||
for _, object := range list.Contents {
|
||||
if object.LastModified == nil || !object.LastModified.Before(cutoff) {
|
||||
continue
|
||||
}
|
||||
if _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: object.Key,
|
||||
}); err != nil {
|
||||
return deleted, fmt.Errorf("delete %s: %w", aws.ToString(object.Key), err)
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
if list.IsTruncated == nil || !*list.IsTruncated {
|
||||
return deleted, nil
|
||||
}
|
||||
token = list.NextContinuationToken
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) key(name, ext string) string {
|
||||
if s.prefix == "" {
|
||||
return name + "." + ext
|
||||
}
|
||||
return s.prefix + "/" + name + "." + ext
|
||||
}
|
||||
176
internal/files/files_test.go
Normal file
176
internal/files/files_test.go
Normal file
@@ -0,0 +1,176 @@
|
||||
package files
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestPutUploadsUniqueUUIDObjectsWithPresignedURLs(t *testing.T) {
|
||||
var puts atomic.Int32
|
||||
keys := make(chan string, 4)
|
||||
client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
status := http.StatusOK
|
||||
switch request.Method {
|
||||
case http.MethodPut:
|
||||
puts.Add(1)
|
||||
keys <- request.URL.Path
|
||||
}
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Status: http.StatusText(status),
|
||||
Header: make(http.Header),
|
||||
Body: io.NopCloser(strings.NewReader("")),
|
||||
Request: request,
|
||||
}, nil
|
||||
})}
|
||||
|
||||
store, err := New(context.Background(), Config{
|
||||
Endpoint: "https://s3.internal",
|
||||
PublicEndpoint: "https://s3.example",
|
||||
Region: "us-east-1",
|
||||
Bucket: "latexgrambot",
|
||||
AccessKeyID: "access",
|
||||
SecretAccessKey: "secret",
|
||||
Prefix: "content",
|
||||
PresignTTL: time.Hour,
|
||||
HTTPClient: client,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
for range 2 {
|
||||
signed, err := store.Put(context.Background(), "jpg", "image/jpeg", []byte("image"))
|
||||
if err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
parsed, err := url.Parse(signed)
|
||||
if err != nil {
|
||||
t.Fatalf("parse signed URL: %v", err)
|
||||
}
|
||||
base := strings.TrimSuffix(strings.TrimPrefix(parsed.Path, "/latexgrambot/content/"), ".jpg")
|
||||
if _, err := uuid.Parse(base); err != nil || uuid.MustParse(base).Version() != 4 {
|
||||
t.Errorf("object name %q is not a UUIDv4", base)
|
||||
}
|
||||
if seen[parsed.Path] {
|
||||
t.Errorf("object name %q was reused", parsed.Path)
|
||||
}
|
||||
seen[parsed.Path] = true
|
||||
if parsed.Query().Get("X-Amz-Signature") == "" || parsed.Query().Get("X-Amz-Expires") != "3600" {
|
||||
t.Errorf("URL is not signed for one hour: %s", signed)
|
||||
}
|
||||
}
|
||||
if got := puts.Load(); got != 2 {
|
||||
t.Fatalf("PUT requests = %d, want 2 (one object per request)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteExpired(t *testing.T) {
|
||||
var deleted []string
|
||||
client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
header := make(http.Header)
|
||||
body := ""
|
||||
status := http.StatusOK
|
||||
switch request.Method {
|
||||
case http.MethodGet: // ListObjectsV2
|
||||
header.Set("Content-Type", "application/xml")
|
||||
body = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>latexgrambot</Name><Prefix>content/</Prefix><KeyCount>2</KeyCount><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>
|
||||
<Contents><Key>content/old.jpg</Key><LastModified>2000-01-01T00:00:00.000Z</LastModified><ETag>"a"</ETag><Size>1</Size><StorageClass>STANDARD</StorageClass></Contents>
|
||||
<Contents><Key>content/fresh.pdf</Key><LastModified>2099-01-01T00:00:00.000Z</LastModified><ETag>"b"</ETag><Size>1</Size><StorageClass>STANDARD</StorageClass></Contents>
|
||||
</ListBucketResult>`
|
||||
case http.MethodDelete:
|
||||
deleted = append(deleted, request.URL.Path)
|
||||
status = http.StatusNoContent
|
||||
}
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Status: http.StatusText(status),
|
||||
Header: header,
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
Request: request,
|
||||
}, nil
|
||||
})}
|
||||
|
||||
store, err := New(context.Background(), Config{
|
||||
Endpoint: "https://s3.internal",
|
||||
PublicEndpoint: "https://s3.example",
|
||||
Region: "us-east-1",
|
||||
Bucket: "latexgrambot",
|
||||
AccessKeyID: "access",
|
||||
SecretAccessKey: "secret",
|
||||
Prefix: "content",
|
||||
PresignTTL: time.Hour,
|
||||
HTTPClient: client,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
|
||||
removed, err := store.DeleteExpired(context.Background(), time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("DeleteExpired: %v", err)
|
||||
}
|
||||
if removed != 1 {
|
||||
t.Fatalf("removed = %d, want 1", removed)
|
||||
}
|
||||
if len(deleted) != 1 || !strings.HasSuffix(deleted[0], "/content/old.jpg") {
|
||||
t.Fatalf("deleted = %v, want the expired object only", deleted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3Integration(t *testing.T) {
|
||||
endpoint := os.Getenv("LATEXGRAMBOT_S3_TEST_ENDPOINT")
|
||||
accessKey := os.Getenv("LATEXGRAMBOT_S3_TEST_ACCESS_KEY")
|
||||
secretKey := os.Getenv("LATEXGRAMBOT_S3_TEST_SECRET_KEY")
|
||||
if endpoint == "" || accessKey == "" || secretKey == "" {
|
||||
t.Skip("S3 integration environment is not configured")
|
||||
}
|
||||
store, err := New(context.Background(), Config{
|
||||
Endpoint: endpoint,
|
||||
PublicEndpoint: endpoint,
|
||||
Region: "us-east-1",
|
||||
Bucket: "latexgrambot",
|
||||
AccessKeyID: accessKey,
|
||||
SecretAccessKey: secretKey,
|
||||
Prefix: "integration",
|
||||
PresignTTL: 5 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
const payload = "latexgrambot S3 integration probe\n"
|
||||
signed, err := store.Put(context.Background(), "txt", "text/plain", []byte(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
response, err := http.Get(signed)
|
||||
if err != nil {
|
||||
t.Fatalf("GET presigned URL: %v", err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
body, err := io.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("read presigned response: %v", err)
|
||||
}
|
||||
if response.StatusCode != http.StatusOK || string(body) != payload {
|
||||
t.Fatalf("presigned GET = %s, body %q", response.Status, body)
|
||||
}
|
||||
}
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||
return f(request)
|
||||
}
|
||||
Reference in New Issue
Block a user