/*
 * This file is part of the Amalthea library.
 * Copyright (C) 2019-2026 Eugene 'Vindex' Stulin
 * Distributed under the BSL 1.0 or the GNU LGPL 3.0 or later.
 */

/**
 * The module implements recognition of file formats.
 *
 * Main function of this module:
 *     `FileFormat getFileFormat(string filepath)`
 * Recognition is carried out according to a complex algorithm based on
 * a special file containing information about file formats.
 * The default path of such file for a normal library installation is:
 *     `/etc/amalthea-${DC}/fileformats.json`
 * You can expand the search paths for format descriptions with:
 *     `void addNewPathToFindFileFormats(string dir)`
 */
module amalthea.fileformats;


import
    std.algorithm,
    std.array,
    std.datetime,
    std.exception,
    std.file,
    std.format,
    std.json,
    std.path,
    std.process,
    std.stdio,
    std.string,
    std.typecons,
    std.utf;

import amalthea.libcore;
import amalthea.dataprocessing : calcIntersection, copyArray;
import amalthea.fs : getRegularFiles, isSymlinkToDir;

pragma(lib, "gio-2.0");
pragma(lib, "glib-2.0");


struct Signature {
    /// Key byte sequence.
    ubyte[] bytes;
    /// Offset of the signature from the beginning of the file.
    uint offset;

    this(ref return scope const Signature rhs) {
        this.bytes = rhs.bytes.dup;
        this.offset = rhs.offset;
    }
}


/*******************************************************************************
 * File format info.
 */
struct FileFormat {
    /// Format name.
    string format;
    /// Key byte sequences and offsets.
    Signature[] signatures;
    /// List of file suffixes used by the format.
    string[] extensions;
    /// Format brief description.
    string description;
    /// Big description. Reserved for future use.
    string extendedDescription;
    /// Mark: plain text or not.
    bool utfText;
    /// Format group name (Audio, Pictures, Archives, etc.).
    string group;
    /** If true, then the format is valid only
     *  if both the extension and the signature match.
     */
    bool completeMatchOnly;
    /// String with shell code for checking.
    string detectionByScript;

    /// Implements comparison.
    int opCmp(in FileFormat rhs) const {
        if (this.format > rhs.format) {
            return 1;
        }
        if (this.format < rhs.format) {
            return -1;
        }
        return 0;
    }

    private void copy(ref scope const FileFormat rhs) {
        this.format = rhs.format.idup;
        this.signatures = copyArray(rhs.signatures);
        this.extensions = rhs.extensions.dup;
        this.description = rhs.description.dup;
        this.extendedDescription = rhs.extendedDescription.dup;
        this.utfText = rhs.utfText;
        this.group = rhs.group.idup;
        this.completeMatchOnly = rhs.completeMatchOnly;
        this.detectionByScript = rhs.detectionByScript;
    }

    this(ref return scope const FileFormat rhs) {
        copy(rhs);
    }

    ref FileFormat opAssign(ref return scope const FileFormat rhs) {
        copy(rhs);
        return this;
    }
}


/*******************************************************************************
 * The function gets the FileFormat structure with info about file format.
 *
 * Params:
 *     filepath = Path to required file.
 *     additExt = File extension as hint for recognition algorithm.
 *                Not needed in most cases. Can be used if for some reason
 *                the file name does not have an extension.
 *
 * Returns: a FileFormat object containing some format information.
 */
FileFormat getFileFormat(string filepath, string additExt) {
    if (!filepath.exists || !filepath.isFile) {
        string msg = "File is not regular or not readable.";
        throw new FileException(filepath, msg);
    }
    return signatureSearch(filepath, additExt);
}


/*******************************************************************************
 * The function gets the FileFormat structure with info about file format.
 *
 * Params:
 *     filepath = Path to required file.
 *
 * Returns: a FileFormat object containing some format information.
 */
FileFormat getFileFormat(string filepath) {
    return getFileFormat(filepath, "");
}


/*******************************************************************************
 * The function returns true if specified file contains plain Unicode-text.
 */
bool isUnicodeTextFile(string filepath) nothrow {
    try {
        File(filepath, "r").byLine.each!validate;
    } catch (Exception e) {
        return false;
    }
    return true;
}


private FileFormat[] searchByExtHint(
    const FileFormat[] candidates, string additExt
) {
    additExt = additExt.toLower;
    FileFormat[] newCandidatesWithExtensions;
    FileFormat[] newCandidatesWithoutExtensions;
    foreach(ftype; candidates) {
        foreach(ext; ftype.extensions) {
            if (ext == "*") {
                newCandidatesWithoutExtensions ~= cast(FileFormat)ftype;
            } else if (additExt == ext) {
                newCandidatesWithExtensions ~= cast(FileFormat)ftype;
            }
        }
    }
    return newCandidatesWithoutExtensions ~ newCandidatesWithExtensions;
}


private FileFormat[] searchByExtensions(
    const FileFormat[] candidates, string fpath
) {
    fpath = fpath.toLower;
    FileFormat[] newCandidatesWithExtensions;
    FileFormat[] newCandidatesWithoutExtensions;
    foreach(ftype; candidates) {
        foreach(ext; ftype.extensions) {
            auto end = "." ~ toLower(ext);
            if (ext == "*") {
                newCandidatesWithoutExtensions ~= cast(FileFormat)ftype;
            } else if (fpath.endsWith(end) || fpath.canFind(end ~ ".")) {
                newCandidatesWithExtensions ~= cast(FileFormat)ftype;
            }
        }
        if (ftype.extensions.empty) {
            newCandidatesWithoutExtensions ~= cast(FileFormat)ftype;
        }
    }
    return newCandidatesWithoutExtensions ~ newCandidatesWithExtensions;
}


static FileFormat[] searchBySignature(
    const FileFormat[] candidates,
    const ubyte[] data
) {
    FileFormat[] newCandidates;
    foreach(ftype; candidates) {
        if (ftype.signatures.length == 0) {
            newCandidates ~= cast(FileFormat)ftype;
        }
        foreach(s; ftype.signatures) {
            auto off = s.offset;
            auto signature = s.bytes;
            if (signature.length + off > data.length) {
                continue;
            }
            if (signature.empty) {
                newCandidates ~= cast(FileFormat)ftype;
                break;
            } else if (signature == data[off .. off + signature.length]) {
                newCandidates ~= cast(FileFormat)ftype;
                break;
            }
        }
    }
    return newCandidates;
}


private FileFormat identifyLikelyFormat(
    const FileFormat[] candidates,
    string filepath
)
in {
    assert(candidates.length > 1);
}
do {
    size_t getMaxNumberOfBytes(const Signature[] signatures) {
        size_t maxNumber;
        foreach(s; signatures) {
            if (s.bytes.length > maxNumber) {
                maxNumber = s.bytes.length;
            }
        }
        return maxNumber;
    }
    FileFormat result = candidates[0];
    FileFormat[] newCandidates = [result];
    size_t maxNumberOfBytes = getMaxNumberOfBytes(result.signatures);
    size_t currentFormatMaxSignatureLength;
    foreach(f; candidates[1 .. $]) {
        currentFormatMaxSignatureLength = getMaxNumberOfBytes(f.signatures);
        if (currentFormatMaxSignatureLength > maxNumberOfBytes) {
            maxNumberOfBytes = currentFormatMaxSignatureLength;
            result = f;
            newCandidates = [result];
        } else if (currentFormatMaxSignatureLength == maxNumberOfBytes) {
            result = f;
            newCandidates ~= result;
        }
    }
    // candidates with the same signature lengths
    if (newCandidates.length > 1) {
        size_t currentMaxExtensionLength;
        foreach(ftype; newCandidates) {
            foreach(ext; ftype.extensions) {
                auto end = "." ~ ext;
                if (filepath.endsWith(end)) {
                    if (currentMaxExtensionLength < ext.length) {
                        currentMaxExtensionLength = ext.length;
                        result = ftype;
                    }
                }
            }
        }
    }
    return result;
}


private bool identifyFacelessFile(string filepath) {
    if (std.path.extension(filepath) == "" && std.file.getSize(filepath) == 0) {
        return true;
    }
    return false;
}


private FileFormat getPlainTextFormat() {
    return FileFormat(
        "TXT", null, null, "Plain text document", "", true, "Text"
    );
}


private FileFormat[] filterFormatsByTextMark(
    const FileFormat[] formats, string filepath
) {
    FileFormat[] result;
    foreach(f; formats) {
        FileFormat newCandidate = f;  // for correct copying to array
        if (!f.utfText || (f.utfText && isUnicodeTextFile(filepath))) {
            result ~= newCandidate;
        }
    }
    return result;
}


private FileFormat[] filterFormatsByDetectionWithScript(
    const FileFormat[] formats, string filepath
) {
    FileFormat[] result;
    foreach(f; formats) {
        FileFormat newCandidate = f;
        if (f.detectionByScript.empty) {
            result ~= newCandidate;
        } else {
            import amalthea.dataprocessing : escapeString;
            string command = f.detectionByScript.replace(
                "%f",
                `"` ~ escapeString(filepath) ~ `"`
            );
            auto shellResult = std.process.executeShell(command);
            if (shellResult.status == 0) {
                return [newCandidate];  // yes, single result
            }
        }
    }
    return result;
}


private FileFormat signatureSearch(string filepath, string additExt = "") {
    auto f = File(filepath, "r");
    ubyte[] buffer;
    buffer.length = 1024;
    ubyte[] data = f.rawRead(buffer);  // todo: mode with O_NOATIME

    FileFormat[] signCandidates = searchBySignature(fileformats, data);
    FileFormat[] extCandidates = additExt.empty ?
        searchByExtensions(fileformats, filepath) :
        searchByExtHint(fileformats, additExt);
    extCandidates = filterFormatsByTextMark(extCandidates, filepath);
    FileFormat[] candidates = calcIntersection(signCandidates, extCandidates);
    candidates = filterFormatsByDetectionWithScript(candidates, filepath);
    if (candidates.empty) {
        signCandidates = signCandidates
            .filter!(c => !c.signatures.empty && !c.completeMatchOnly)
            .array;
        if (signCandidates.empty) {
            return altFormatSearch(filepath);
        } else if (signCandidates.length == 1) {
            return signCandidates[0];
        }
        return identifyLikelyFormat(signCandidates, filepath);
    }
    if (candidates.length == 1) {
        return candidates[0];
    }
    // where candidates.length > 1
    return identifyLikelyFormat(candidates, filepath);
}


extern(C)
static char* g_content_type_guess(
    const char* filename,
    const byte* data,
    size_t data_size,
    bool* result_uncertain
);

extern(C)
static void g_free(void* mem);

extern(C)
char* g_content_type_get_mime_type(const char* type);

extern(C)
char* g_content_type_get_description(const char* type);


/// Returns tuple with MIME type and its description.
auto contentTypeGuess(alias useEnglish=true)(string filepath) {
    static if (useEnglish) {
        auto lang = environment.get("LANG", "");
        environment["LANG"] = "C";
        scope(exit) environment["LANG"] = lang;
    }

    auto raw = cast(byte[])read(filepath, 1024);
    byte* data = raw.ptr;
    char* type = g_content_type_guess(
        filepath.toStringz, data, raw.length, null
    );
    string mimetype = type.fromStringz.idup;
    char* descr = g_content_type_get_description(type);
    string description = descr.fromStringz.idup;
    g_free(type);
    g_free(descr);
    return tuple(mimetype, description);
}


private FileFormat altFormatSearch(string filepath) {
    if (identifyFacelessFile(filepath)) {
        return getPlainTextFormat();
    }
    auto mimeAndDescription = contentTypeGuess(filepath);
    string mimetype = mimeAndDescription[0];
    string mimegroup = mimetype.split('/')[0];
    string mimeformat = mimetype.split('/')[1];
    string description = mimeAndDescription[1];

    FileFormat ff;
    ff.format = mimeformat;
    ff.description = description.capitalize;
    if (ff.format == "plain") {
        return getPlainTextFormat();
    } else if (ff.format == "x-makefile") {
        ff.format = "Makefile";
    } else if (mimeformat.endsWith("src")) {
        ff.format = ff.description;
    }
    if (isUnicodeTextFile(filepath) || mimegroup == "text") {
        ff.group = "Text";
    }
    return ff;
}


/*******************************************************************************
 * The functions allows to extend paths to find JSON files with file formats.
 *
 * Params:
 *     dir = Directory to search for JSON files.
 */
void addNewPathToFindFileFormats(string dir) {
    enforce(
        exists(dir) && (isDir(dir) || isSymlinkToDir(dir)),
        new FileException(dir, ": directory not found.")
    );
    auto entries = getRegularFiles(dir, No.hidden)
        .filter!(a => a.path.extension.toLower == ".json");
    JSONValue[] jsonFormatArray;
    foreach(entry; entries) {
        jsonFormatArray ~= entry.path.readText.parseJSON.array;
    }
    extendFileFormatBase(jsonFormatArray);
}


immutable string canonBase;
shared static this() {
    enum fileformatsJSONText = import("res/fileformats.json");
    canonBase = format!"/etc/amalthea-%s/fileformats.json"(amaltheaCompiler);
    JSONValue[] jsonFormatArray;
    if (exists(canonBase)) {
        string text = readText(canonBase);
        jsonFormatArray = parseJSON(text).array;
    } else {
        jsonFormatArray = parseJSON(fileformatsJSONText).array;
    }
    collectException(extendFileFormatBase(jsonFormatArray));
}


private __gshared FileFormat[] fileformats;

/// Returns array of supported file formats.
FileFormat[] getAllFileFormats() {
    return copyArray(fileformats);
}


private void extendFileFormatBase(JSONValue[] jsonFormatArray) {
    auto spec = singleSpec("%x");
    foreach(j; jsonFormatArray) {
        FileFormat newFormat;
        newFormat.format = j["format"].str;
        if ("signatures" in j) {
            foreach(s; j["signatures"].array) {
                Signature newSignature;
                newSignature.offset = cast(uint)s["offset"].integer;
                foreach(strByte; s["hex_signature"].str.split(' ')) {
                    newSignature.bytes ~= unformatValue!ubyte(strByte, spec);
                }
                newFormat.signatures ~= newSignature;
            }
        }
        if ("extensions" in j) {
            foreach(ext; j["extensions"].array) {
                newFormat.extensions ~= ext.str;
            }
        }
        if ("complete_match" in j) {
            newFormat.completeMatchOnly = j["complete_match"].boolean;
        }
        if ("description" in j) {
            newFormat.description = j["description"].str;
        }
        if ("extended_description" in j) {
            newFormat.extendedDescription = j["extended_description"].str;
        }
        if ("group" in j) {
            newFormat.group = j["group"].str;
        }
        if ("detection_by_script" in j) {
            newFormat.detectionByScript = j["detection_by_script"].str;
        }
        fileformats ~= newFormat;  // global variable
    }
}
