Initial commit: DSH Web Manager
This commit is contained in:
@@ -0,0 +1,252 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.misaka.ren/m1saka/dsh-web/internal/docker"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
var npmBuildClient = &http.Client{Timeout: 15 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
||||
|
||||
const imageRecipeRev = "7"
|
||||
|
||||
var imagePlatform = runtime.GOOS + "/" + runtime.GOARCH
|
||||
|
||||
const minimumBuildDiskBytes = uint64(2 << 30)
|
||||
|
||||
var buildDiskAvailable = func(path string) (uint64, error) {
|
||||
var stat syscall.Statfs_t
|
||||
if err := syscall.Statfs(path, &stat); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return uint64(stat.Bavail) * uint64(stat.Bsize), nil
|
||||
}
|
||||
|
||||
type inspectedImage struct {
|
||||
ID string `json:"Id"`
|
||||
Os, Architecture, Variant, Created string
|
||||
RepoTags []string
|
||||
Config struct{ Labels map[string]string }
|
||||
}
|
||||
|
||||
func (im inspectedImage) platform() string {
|
||||
if im.Os == "" || im.Architecture == "" {
|
||||
return ""
|
||||
}
|
||||
p := im.Os + "/" + im.Architecture
|
||||
if im.Variant != "" {
|
||||
p += "/" + im.Variant
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func (a *API) inspectLocalImage(ctx context.Context, id string) (*inspectedImage, error) {
|
||||
if id == "" {
|
||||
return nil, nil
|
||||
}
|
||||
var im inspectedImage
|
||||
if err := a.docker.Do(ctx, http.MethodGet, "/images/"+url.PathEscape(id)+"/json", nil, &im); err != nil {
|
||||
var apiErr *docker.APIError
|
||||
if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &im, nil
|
||||
}
|
||||
|
||||
func (a *API) imageMatches(ctx context.Context, id, version string) (bool, error) {
|
||||
im, err := a.inspectLocalImage(ctx, id)
|
||||
if err != nil || im == nil {
|
||||
return false, err
|
||||
}
|
||||
return im.ID == id && im.platform() == imagePlatform && im.Config.Labels["dsh-mgr.role"] == "runtime" && im.Config.Labels["dsh-mgr.version"] == version && im.Config.Labels["dsh-mgr.recipe"] == imageRecipeRev, nil
|
||||
}
|
||||
|
||||
func (a *API) imageIndex(ctx context.Context) ([]imageRecord, error) {
|
||||
buildQueue.Lock()
|
||||
defer buildQueue.Unlock()
|
||||
if err := a.ensureBuilds(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := a.s.DB.ExecContext(ctx, `INSERT OR IGNORE INTO images(image_id,version,recipe_rev,platform,tag,status,built_at)
|
||||
SELECT image_id,version,recipe_rev,platform,'','missing',COALESCE(finished_at,created_at) FROM image_builds WHERE status IN ('success','missing') AND image_id<>'' ORDER BY created_at DESC,id DESC`); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := a.s.DB.QueryContext(ctx, `SELECT image_id,version,COALESCE(recipe_rev,''),COALESCE(platform,''),tag,status,built_at,
|
||||
(SELECT count(*) FROM instances WHERE instances.image_id=images.image_id) FROM images ORDER BY built_at DESC,image_id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := []imageRecord{}
|
||||
for rows.Next() {
|
||||
var im imageRecord
|
||||
if err = rows.Scan(&im.ImageID, &im.Version, &im.RecipeRev, &im.Platform, &im.Tag, &im.Status, &im.Created, &im.UsageCount); err != nil {
|
||||
break
|
||||
}
|
||||
im.ID = im.ImageID
|
||||
result = append(result, im)
|
||||
}
|
||||
scanErr := rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if scanErr != nil {
|
||||
return nil, scanErr
|
||||
}
|
||||
filters, _ := json.Marshal(map[string][]string{"label": {"dsh-mgr.role=runtime"}})
|
||||
var local []struct {
|
||||
ID string
|
||||
Labels map[string]string
|
||||
}
|
||||
if err := a.docker.Do(ctx, http.MethodGet, "/images/json?filters="+url.QueryEscape(string(filters)), nil, &local); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
known := make(map[string]bool, len(result))
|
||||
for _, im := range result {
|
||||
known[im.ImageID] = true
|
||||
}
|
||||
for _, im := range local {
|
||||
if im.ID != "" && !known[im.ID] && im.Labels["dsh-mgr.role"] == "runtime" {
|
||||
known[im.ID] = true
|
||||
result = append(result, imageRecord{ID: im.ID, ImageID: im.ID})
|
||||
}
|
||||
}
|
||||
for i := range result {
|
||||
im := &result[i]
|
||||
im.Status = "missing"
|
||||
actual, err := a.inspectLocalImage(ctx, im.ImageID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if actual == nil {
|
||||
continue
|
||||
}
|
||||
im.Status = "unverified"
|
||||
im.Platform = actual.platform()
|
||||
im.Version = actual.Config.Labels["dsh-mgr.version"]
|
||||
im.RecipeRev = actual.Config.Labels["dsh-mgr.recipe"]
|
||||
im.Tag = ""
|
||||
if len(actual.RepoTags) > 0 {
|
||||
im.Tag = actual.RepoTags[0]
|
||||
}
|
||||
if actual.Created != "" {
|
||||
im.Created = actual.Created
|
||||
}
|
||||
if err := a.s.DB.QueryRowContext(ctx, "SELECT count(*) FROM instances WHERE image_id=?", im.ImageID).Scan(&im.UsageCount); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var verified int
|
||||
if err := a.s.DB.QueryRowContext(ctx, "SELECT count(*) FROM image_builds WHERE image_id=? AND version=? AND recipe_rev=? AND platform=? AND status='success'", im.ImageID, im.Version, imageRecipeRev, imagePlatform).Scan(&verified); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if verified > 0 && actual.ID == im.ImageID && actual.Config.Labels["dsh-mgr.role"] == "runtime" && im.RecipeRev == imageRecipeRev && im.Platform == imagePlatform {
|
||||
im.Status = "available"
|
||||
}
|
||||
}
|
||||
tx, err := a.s.DB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
for _, im := range result {
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO images(image_id,version,recipe_rev,platform,tag,status,built_at,usage_count) VALUES(?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(image_id) DO UPDATE SET version=excluded.version,recipe_rev=excluded.recipe_rev,platform=excluded.platform,tag=excluded.tag,status=excluded.status,built_at=excluded.built_at,usage_count=excluded.usage_count`, im.ImageID, im.Version, im.RecipeRev, im.Platform, im.Tag, im.Status, im.Created, im.UsageCount); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if im.Status == "missing" {
|
||||
if _, err = tx.ExecContext(ctx, "UPDATE image_builds SET status='missing',error='image no longer available or template identity is unknown' WHERE image_id=? AND status='success'", im.ImageID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return result, tx.Commit()
|
||||
}
|
||||
|
||||
func (a *API) StartImageBuilds() error {
|
||||
buildQueue.Lock()
|
||||
defer buildQueue.Unlock()
|
||||
if err := a.ensureBuilds(); err != nil {
|
||||
return err
|
||||
}
|
||||
a.startBuildWorker()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *API) checkBuildDisk(ctx context.Context) error {
|
||||
var info struct{ DockerRootDir string }
|
||||
if err := a.docker.Do(ctx, http.MethodGet, "/info", nil, &info); err != nil {
|
||||
return fmt.Errorf("build disk check: %w", err)
|
||||
}
|
||||
// A matching pathname alone cannot establish daemon filesystem identity.
|
||||
confirmed := os.Getenv("DSH_DOCKER_ROOT_VISIBLE")
|
||||
if !filepath.IsAbs(info.DockerRootDir) || confirmed != info.DockerRootDir {
|
||||
return errors.New("Docker disk capacity unknown: mount the daemon DockerRootDir at the identical path and set DSH_DOCKER_ROOT_VISIBLE to that path; new builds are blocked")
|
||||
}
|
||||
available, err := buildDiskAvailable(confirmed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Docker disk capacity unavailable: %w", err)
|
||||
}
|
||||
return requireBuildDisk(available)
|
||||
}
|
||||
|
||||
func requireBuildDisk(available uint64) error {
|
||||
if available < minimumBuildDiskBytes {
|
||||
return fmt.Errorf("insufficient Docker disk space: %d available bytes, require %d", available, minimumBuildDiskBytes)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *API) insertBuildRequest(b buildRecord, user string) error {
|
||||
tx, err := a.s.DB.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err = tx.Exec("INSERT INTO image_builds(id,version,status,created_at,requested_by,recipe_rev,platform) VALUES(?,?,?,?,?,?,?)", b.ID, b.Version, b.Status, b.Created, b.RequestedBy, b.RecipeRev, b.Platform); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.Exec("INSERT INTO image_build_requests(user_id,build_id,created_at) VALUES(?,?,?)", user, b.ID, b.Created); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (a *API) reconcileBuilds(parent context.Context, records []buildRecord) error {
|
||||
ctx, cancel := context.WithTimeout(parent, 15*time.Second)
|
||||
defer cancel()
|
||||
for i := range records {
|
||||
b := &records[i]
|
||||
if b.Status != "success" {
|
||||
continue
|
||||
}
|
||||
if b.RecipeRev != imageRecipeRev || b.Platform != imagePlatform {
|
||||
b.Status, b.Error = "missing", "image template identity is unknown"
|
||||
if _, err := a.s.DB.Exec("UPDATE image_builds SET status=?,error=? WHERE id=? AND status='success'", b.Status, b.Error, b.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
id, err := a.localBuildImage(ctx, b.Version, b.ImageID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if id == "" || id != b.ImageID {
|
||||
b.Status, b.Error = "missing", "image no longer available"
|
||||
if _, err := a.s.DB.Exec("UPDATE image_builds SET status=?,error=? WHERE id=? AND status='success'", b.Status, b.Error, b.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user