package fileedit import ( "archive/zip" "crypto/rand" "encoding/hex" "errors" "fmt" "io" "io/fs" "math" "os" "path" "sort" "strings" "unicode/utf8" "golang.org/x/text/encoding/simplifiedchinese" ) // The codes an unzip refuses an archive with. Each names what is wrong with the // ARCHIVE, so the panel can say "this zip is broken" rather than "your path is // wrong"; Result.Entry names the entry. const ( // CodeArchiveInvalid is a file that is not a zip, a damaged one, one whose // entry bytes disagree with their header (size or CRC), or one with nothing in it. CodeArchiveInvalid = "archive_invalid" // CodeArchiveUnsafe is an entry naming a path outside the folder it is // extracted into ("../", "/etc/x", "C:\x"), or a device, pipe or socket. CodeArchiveUnsafe = "archive_unsafe" // CodeArchiveSymlink is an entry that is a symbolic link. A link extracted into // the world could point anywhere, and every later op would have to reason about // it, so an archive carrying one is refused whole. CodeArchiveSymlink = "archive_symlink" // CodeTypeConflict is an archive with a file where the server has a folder, a // folder where it has a file, or anything where it has a link. Overwrite never // resolves it: replacing a folder with a file would delete the folder. CodeTypeConflict = "type_conflict" ) // MaxConflicts bounds Result.Conflicts, keeping the result line bounded when an // archive would replace a whole world; ConflictCount still says how many there // are. maxConflictBytes bounds the names listed as well: felis-api reads an // unzip's result from the last opLogLines lines of its Pod's log, and the // container runtime splits a line longer than 16 KiB into several, so 200 long // paths would push the result marker out of that tail and the op would read as // ended without a result. A file that exists has a path under PATH_MAX (4 KiB), // so the first conflict always fits. const ( MaxConflicts = 200 maxConflictBytes = 8 << 10 ) // unzipEntryOverhead is what the space check adds per entry for the inode and // directory block it takes beyond its bytes. const unzipEntryOverhead = 4096 // unzipTempPrefix names the working folder an unzip extracts into. It sits in // the destination folder, so every move out of it is a rename on one volume. const unzipTempPrefix = ".felis-unzip-" // unzip extracts the .zip at name into the folder holding it. // // It is all or nothing. Every check that can refuse the archive runs before a // byte is written: entry names, entry types, what is already on the server, and // the room on the volume. The entries are then extracted into a working folder // beside the destination, and only once every one of them has been written and // verified are they renamed into place. A failure at any point before that // leaves the destination exactly as it was. The renames themselves are journaled // and undone in reverse if one fails, so a replaced file comes back. Only a Job // killed in the middle of the renames — a few milliseconds for thousands of // files — can leave the archive half applied. // // An existing file the archive would replace is a conflict: without overwrite the // unzip lists them (CodeExists, Conflicts) and changes nothing; with it they are // replaced. Folders merge. A file where the server has a folder, or the reverse, // is CodeTypeConflict whatever overwrite says. // // The only size bound is the volume. Each entry's declared size is summed and // checked against the free space up front, and archive/zip itself refuses an // entry whose bytes run past its declared size or fail its CRC, so an archive // that lies about its sizes (a zip bomb) stops at the first lying entry and // nothing it wrote survives. func unzip(r *os.Root, rootPath, name string, overwrite bool, progress func(done, total int64)) Result { name = path.Clean(name) if !strings.EqualFold(path.Ext(name), ".zip") { return Result{Code: CodeBadPath, Error: fmt.Sprintf("%s is not a .zip archive", name)} } f, err := r.Open(name) if err != nil { return failure(err, name) } defer f.Close() info, err := f.Stat() if err != nil { return failure(err, name) } if !info.Mode().IsRegular() { return Result{Code: CodeBadPath, Error: fmt.Sprintf("%s is not a file", name)} } // ErrInsecurePath comes back WITH a usable reader, and only under // GODEBUG=zipinsecurepath=0; planUnzip does that check itself, for every // entry, whatever the setting. zr, err := zip.NewReader(f, info.Size()) if err != nil && !errors.Is(err, zip.ErrInsecurePath) { return Result{Code: CodeArchiveInvalid, Error: fmt.Sprintf("%s is not a readable zip archive: %v", name, err)} } p, res := planUnzip(zr.File) if res.Code != "" { return res } dest := path.Dir(name) present, replaced, res := checkTargets(r, dest, p) if res.Code != "" { return res } if len(replaced) > 0 && !overwrite { list := make([]string, 0, len(replaced)) for n := range replaced { list = append(list, path.Join(dest, n)) } sort.Strings(list) count, n, size := len(list), 0, 0 for n < count && n < MaxConflicts && size+len(list[n]) <= maxConflictBytes { size += len(list[n]) n++ } list = list[:n] return Result{Code: CodeExists, Conflicts: list, ConflictCount: count, Error: fmt.Sprintf( "%d files in the archive already exist on the server; extract again with overwrite to replace them", count)} } // A working folder left by an unzip that was killed holds only a copy, and // clearing it first gives its room back to the check below. sweepUnzipTemps(r, dest) need := p.bytes + int64(len(p.files)+len(p.dirs))*unzipEntryOverhead if avail, _, err := statfs(rootPath); err == nil && uint64(need) > avail { free := int64(math.MaxInt64) if avail < math.MaxInt64 { free = int64(avail) } return Result{Code: CodeNoSpace, Need: need, Avail: free, Error: fmt.Sprintf( "extracting %s needs %d bytes and the server's volume has %d free; nothing was changed", name, need, free)} } var suffix [6]byte if _, err := rand.Read(suffix[:]); err != nil { return Result{Code: CodeBadPath, Error: fmt.Sprintf("generate a temporary name: %v", err)} } tmp := path.Join(dest, unzipTempPrefix+hex.EncodeToString(suffix[:])) staged, old := path.Join(tmp, "new"), path.Join(tmp, "old") for _, d := range []string{tmp, staged, old} { if err := r.Mkdir(d, 0o700); err != nil { _ = r.RemoveAll(tmp) return unzipWriteFailure(err, dest) } } // After a success tmp holds only the files the archive replaced; after a // failure, everything the archive wrote. Either way it goes. defer func() { _ = r.RemoveAll(tmp) }() if res := extractAll(r, staged, p, progress); res.Code != "" { return res } if res := placeAll(r, dest, staged, old, p, present, replaced); res.Code != "" { return res } return Result{Files: len(p.files), Bytes: p.bytes} } // unzipPlan is an archive's entries once every one has passed planUnzip. Names // are cleaned, slash-separated and relative to the destination folder. type unzipPlan struct { files []zipFile // dirs is every folder the archive makes, named in it or implied by a file // inside it, sorted so a folder comes before everything in it. dirs []string isDir map[string]bool bytes int64 // the declared size of every file, summed } type zipFile struct { f *zip.File name string mode fs.FileMode } // planUnzip checks every entry's name and type and works out what the archive // makes. Nothing about the server is consulted yet. func planUnzip(entries []*zip.File) (unzipPlan, Result) { p := unzipPlan{isDir: map[string]bool{}} for _, f := range entries { raw := entryName(f) name, ok := cleanEntry(raw) if !ok { return p, Result{Code: CodeArchiveUnsafe, Entry: raw, Error: fmt.Sprintf( "%s leads outside the folder it would be extracted into", raw)} } // macOS's Finder adds __MACOSX/ to every zip it makes: resource forks that // mean nothing on the server. if name == "__MACOSX" || strings.HasPrefix(name, "__MACOSX/") { continue } mode := f.Mode() isDir := mode.IsDir() || strings.HasSuffix(raw, "/") switch { case mode&fs.ModeSymlink != 0: return p, Result{Code: CodeArchiveSymlink, Entry: raw, Error: fmt.Sprintf( "%s is a symbolic link; an archive containing links is not extracted", raw)} case isDir: if name != "." { p.isDir[name] = true } continue case !mode.IsRegular(): return p, Result{Code: CodeArchiveUnsafe, Entry: raw, Error: fmt.Sprintf( "%s is not a regular file", raw)} case name == ".": return p, Result{Code: CodeArchiveUnsafe, Entry: raw, Error: fmt.Sprintf( "%s names the destination folder itself", raw)} case f.UncompressedSize64 > uint64(math.MaxInt64-p.bytes): return p, Result{Code: CodeArchiveInvalid, Entry: raw, Error: fmt.Sprintf( "%s declares an impossible size", raw)} } p.bytes += int64(f.UncompressedSize64) // Anything the archive marks executable (a start.sh) stays executable; // every other permission is the server's usual. perm := fs.FileMode(0o644) if mode.Perm()&0o111 != 0 { perm = 0o755 } p.files = append(p.files, zipFile{f: f, name: name, mode: perm}) } // Sorted, a name the archive holds twice sits next to itself; a set of names // would cost as much again as the entries for an archive of many small files. sort.Slice(p.files, func(i, j int) bool { return p.files[i].name < p.files[j].name }) for i := 1; i < len(p.files); i++ { if p.files[i].name == p.files[i-1].name { return p, Result{Code: CodeArchiveInvalid, Entry: p.files[i].name, Error: fmt.Sprintf( "%s appears in the archive twice", p.files[i].name)} } } for _, zf := range p.files { // cleanEntry already refused a rooted name; stopping at "/" as well keeps // this loop finite should that check ever move. for d := path.Dir(zf.name); d != "." && d != "/"; d = path.Dir(d) { p.isDir[d] = true } } for _, zf := range p.files { if p.isDir[zf.name] { return p, Result{Code: CodeArchiveInvalid, Entry: zf.name, Error: fmt.Sprintf( "%s is both a file and a folder in the archive", zf.name)} } } if len(p.files) == 0 && len(p.isDir) == 0 { return p, Result{Code: CodeArchiveInvalid, Error: "the archive has nothing to extract"} } for d := range p.isDir { p.dirs = append(p.dirs, d) } // A folder's name is a prefix of everything in it, and a prefix sorts first. sort.Strings(p.dirs) return p, Result{} } // entryName is an entry's name as its maker meant it. A zip made on Chinese // Windows stores names in the system code page (GBK) without the UTF-8 flag; a // name that is not valid UTF-8 is decoded as GB18030, GBK's superset. The flag // alone is no guide: archive/zip reports NonUTF8 for every name made without it, // which includes plain ASCII and macOS's UTF-8. Decoding comes before the // backslash below because a GBK trail byte can be 0x5C. func entryName(f *zip.File) string { name := f.Name if !utf8.ValidString(name) { if s, err := simplifiedchinese.GB18030.NewDecoder().String(name); err == nil { name = s } } return strings.ReplaceAll(name, `\`, "/") } // cleanEntry cleans an entry name and reports false for one that must not be // extracted: a NUL, an absolute path, a drive letter, or a climb out of the // destination. It judges the name as TEXT, which is sound here because nothing // is resolved through it until checkTargets and extraction, and those go through // os.Root, which would refuse an escape anyway. func cleanEntry(raw string) (string, bool) { if raw == "" || strings.ContainsRune(raw, 0) || strings.HasPrefix(raw, "/") { return "", false } if len(raw) >= 2 && raw[1] == ':' && (raw[0]|0x20) >= 'a' && (raw[0]|0x20) <= 'z' { return "", false } name := path.Clean(raw) if name == ".." || strings.HasPrefix(name, "../") { return "", false } return name, true } // checkTargets looks at what the server already has where the archive lands. // present is the archive's folders that exist on the server (they merge); // replaced is its files that do (conflicts). A folder the server lacks cannot // hold anything, so its contents are not looked up. func checkTargets(r *os.Root, dest string, p unzipPlan) (present, replaced map[string]bool, res Result) { present, replaced = map[string]bool{}, map[string]bool{} absent := func(name string) bool { parent := path.Dir(name) return parent != "." && !present[parent] } for _, d := range p.dirs { if absent(d) { continue } at := path.Join(dest, d) info, err := r.Lstat(at) switch { case errors.Is(err, fs.ErrNotExist): case err != nil: return nil, nil, failure(err, at) case info.Mode()&fs.ModeSymlink != 0: return nil, nil, typeConflict(at, "is a link on the server, and the archive has a folder there") case info.IsDir(): present[d] = true default: return nil, nil, typeConflict(at, "is a file on the server, and the archive has a folder there") } } for _, zf := range p.files { if absent(zf.name) { continue } at := path.Join(dest, zf.name) info, err := r.Lstat(at) switch { case errors.Is(err, fs.ErrNotExist): case err != nil: return nil, nil, failure(err, at) case info.Mode().IsRegular(): replaced[zf.name] = true case info.IsDir(): return nil, nil, typeConflict(at, "is a folder on the server, and the archive has a file there") default: return nil, nil, typeConflict(at, "is a link or special file on the server, and the archive has a file there") } } return present, replaced, Result{} } func typeConflict(at, why string) Result { return Result{Code: CodeTypeConflict, Entry: at, Error: fmt.Sprintf("%s %s; nothing was changed", at, why)} } // extractAll writes every folder and file of p under staged, handed to the game // uid, and syncs each file. func extractAll(r *os.Root, staged string, p unzipPlan, progress func(done, total int64)) Result { for _, d := range p.dirs { at := path.Join(staged, d) if err := r.Mkdir(at, 0o755); err != nil { return unzipWriteFailure(err, d) } _ = ownWritten(r, at) } var done int64 count := func(n int) { done += int64(n) if progress != nil { progress(done, p.bytes) } } for _, zf := range p.files { if res := extractOne(r, path.Join(staged, zf.name), zf, count); res.Code != "" { return res } } for _, d := range p.dirs { syncDir(r, path.Join(staged, d)) } syncDir(r, staged) return Result{} } func extractOne(r *os.Root, at string, zf zipFile, count func(int)) Result { invalid := func(err error) Result { return Result{Code: CodeArchiveInvalid, Entry: zf.name, Error: fmt.Sprintf( "%s in the archive is damaged: %v; nothing was changed", zf.name, err)} } src, err := zf.f.Open() if err != nil { return invalid(err) } defer src.Close() w, err := r.OpenFile(at, os.O_WRONLY|os.O_CREATE|os.O_EXCL, zf.mode) if err != nil { return unzipWriteFailure(err, zf.name) } if err := w.Chmod(zf.mode); err != nil { w.Close() return unzipWriteFailure(err, zf.name) } if _, err := io.Copy(countingWriter{w, count}, archiveReader{src}); err != nil { w.Close() var ae *archiveError if errors.As(err, &ae) { return invalid(ae.err) } return unzipWriteFailure(err, zf.name) } if err := syncWritten(w); err != nil { w.Close() return unzipWriteFailure(err, zf.name) } if err := w.Close(); err != nil { return unzipWriteFailure(err, zf.name) } _ = ownWritten(r, at) return Result{} } // placeAll renames the extracted tree into dest. A folder the server lacks moves // whole; one it has is descended into. A file it has is first moved aside into // old, so undoing the journal puts it back. func placeAll(r *os.Root, dest, staged, old string, p unzipPlan, present, replaced map[string]bool) Result { // Only the destination and the folders the server has are descended into; // everything else moves with the folder it is in, so its name is not kept. kids := map[string][]string{} add := func(name string) { if parent := path.Dir(name); parent == "." || present[parent] { kids[parent] = append(kids[parent], name) } } for _, d := range p.dirs { add(d) } for _, zf := range p.files { add(zf.name) } type move struct{ from, to string } var journal []move mv := func(from, to string) error { if err := renameEntry(r, from, to); err != nil { return err } journal = append(journal, move{from, to}) return nil } var place func(dir string) error place = func(dir string) error { for _, c := range kids[dir] { src, dst := path.Join(staged, c), path.Join(dest, c) switch { case p.isDir[c] && present[c]: if err := place(c); err != nil { return err } case replaced[c]: aside := path.Join(old, c) if err := r.MkdirAll(path.Dir(aside), 0o700); err != nil { return err } if err := mv(dst, aside); err != nil { return err } if err := mv(src, dst); err != nil { return err } default: if err := mv(src, dst); err != nil { return err } } } return nil } if err := place("."); err != nil { for i := len(journal) - 1; i >= 0; i-- { _ = r.Rename(journal[i].to, journal[i].from) } return unzipWriteFailure(err, dest) } syncDir(r, dest) for d := range present { syncDir(r, path.Join(dest, d)) } return Result{} } // renameEntry is the rename placeAll moves with. A var so a test can fail one // part-way through and watch the journal undo the rest. var renameEntry = func(r *os.Root, from, to string) error { return r.Rename(from, to) } // sweepUnzipTemps removes working folders a killed unzip left in dir. func sweepUnzipTemps(r *os.Root, dir string) { d, err := r.Open(dir) if err != nil { return } names, _ := d.Readdirnames(-1) d.Close() for _, n := range names { if isUnzipTemp(n) { _ = r.RemoveAll(path.Join(dir, n)) } } } func isUnzipTemp(name string) bool { suffix, ok := strings.CutPrefix(name, unzipTempPrefix) if !ok || len(suffix) != 12 { return false } _, err := hex.DecodeString(suffix) return err == nil } // unzipWriteFailure is writeFailure worded for an unzip, which by then has // changed nothing whatever the step that failed. func unzipWriteFailure(err error, name string) Result { res := writeFailure(err, name) if res.Code == CodeNoSpace { res.Error = "the server's volume filled up while extracting; nothing was changed" } return res } // archiveError marks a failure reading an entry's bytes out of the archive, so // extractOne can tell a damaged archive from the volume failing underneath. type archiveError struct{ err error } func (e *archiveError) Error() string { return "read archive: " + e.err.Error() } func (e *archiveError) Unwrap() error { return e.err } type archiveReader struct{ r io.Reader } func (a archiveReader) Read(p []byte) (int, error) { n, err := a.r.Read(p) if err != nil && err != io.EOF { err = &archiveError{err} } return n, err } // countingWriter reports each write's length to add. type countingWriter struct { w io.Writer add func(int) } func (c countingWriter) Write(p []byte) (int, error) { n, err := c.w.Write(p) c.add(n) return n, err }