Better error handling & structure

This commit is contained in:
Mehdi Benadel 2023-08-01 07:30:52 +02:00
parent 4ce37bf7b8
commit 0f99a3632b
2 changed files with 280 additions and 191 deletions

View file

@ -6,7 +6,7 @@ readme = "README.md"
homepage = "https://git.joinfirefish.org/firefish/emoji-gen"
repository = "https://git.joinfirefish.org/firefish/emoji-gen"
authors = ["cutestnekoaqua <waterdev@galaxcrow.de>", "Mehdi Benadel <mehdi.benadel@gmail.com>"]
version = "0.3.1"
version = "0.3.2"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@ -24,3 +24,9 @@ regex = "1.9.1"
url = "2.4.0"
reqwest = { version = "0.11.18", features = ["blocking", "json"] }
tempdir = "0.3.7"
error-stack = "0.3.1"
env_logger = "0.10.0"
log = "0.4.19"
[dev-dependencies]
env_logger = "0.10.0"

View file

@ -1,51 +1,45 @@
#![allow(non_snake_case)]
use std::fmt;
use std::fs::File;
use std::path::Path;
use std::path::PathBuf;
use std::io::Read;
use std::io::Seek;
use std::io::Write;
use std::rc::Rc;
use std::error::Error;
use error_stack::{report, Result};
use imghdr::Type;
use imghdr;
use regex::Regex;
use reqwest;
use reqwest::Url;
use tempdir::TempDir;
use clap::{Args, Parser, Subcommand};
use debug_print::debug_println;
use clap::{Parser, Subcommand};
use walkdir::WalkDir;
use walkdir::DirEntry;
use serde::{Deserialize, Serialize};
use zip::result::ZipError;
use zip::write::FileOptions;
use env_logger;
use log::{debug, info, warn, error};
#[derive(Parser, Debug)]
#[command(author, version, about, long_about = None)]
struct Cli {
/// Option
#[command(subcommand)]
command: Option<Command>,
}
#[derive(Debug, Args)]
struct GlobalOpts {
/// Output file path
#[arg(short = 'o', long = "output", default_value_t = String::from("generated_emojis"))]
outputFilepath: String,
command: Command,
}
#[derive(Debug, Subcommand)]
enum Command {
/// Help message for read.
Local {
#[command(flatten)]
global_opts: GlobalOpts,
/// Output file path
#[arg(short = 'o', long = "output", default_value_t = String::from("generated_emojis"))]
outputFilepath: String,
/// Folder with the custom emojis to generate the pack from.
#[arg(short, long)]
@ -61,8 +55,9 @@ enum Command {
},
/// Help message for write.
Crawl {
#[command(flatten)]
global_opts: GlobalOpts,
/// Output file path
#[arg(short = 'o', long = "output", default_value_t = String::from("generated_emojis"))]
outputFilepath: String,
/// Host to crawl emojis from
#[arg(short = 'h', long = "host", default_value_t = String::from("https://firefish.social"))]
@ -125,159 +120,195 @@ fn getTypename(typeEnum: imghdr::Type) -> &'static str {
};
}
fn main() {
let args = Cli::parse();
#[derive(Debug)]
enum EmojiGenError {
ZipCreationFailed,
EmojiFetchFailed,
MetadataGenerationFailed,
ImageFetchingFailed,
}
match args.command {
Some(Command::Local { global_opts, folder, originHost, group }) => {
local(
global_opts.outputFilepath,
originHost,
group,
Path::new(folder.as_str()),
)
},
Some(Command::Crawl { global_opts, host }) => {
crawl(
global_opts.outputFilepath,
&Url::parse(&host).unwrap(),
)
},
None => {
println!("Default subcommand");
}
impl fmt::Display for EmojiGenError {
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt.write_str("Error processing emojis: Could not create the emoji zip bundle.")
}
}
fn local(
outputFilepath: String,
originHost: String,
group: String,
folder : &Path,
) {
impl Error for EmojiGenError {}
fn main() {
env_logger::init();
let args = Cli::parse();
match process(args.command) {
Ok(()) => {},
Err(err) => {
error!("The process could not be completed. Quiting.");
error!("\n{err:?}");
}
};
}
fn process(
command: Command
) -> Result<(), EmojiGenError> {
let tmpDir: TempDir = TempDir::new("emoji-gen").unwrap();
let tmpFolder: &Path = tmpDir.path();
let zipFilepath: PathBuf;
let emojis: Vec<Emoji>;
let hostUrl: String;
folder.canonicalize().unwrap_or_else(| _ | {
println!("Folder path '{}' is invalid.", folder.to_str().unwrap());
std::process::exit(1);
});
let zipFilepath = prepare_zipfile(outputFilepath);
let emojis = get_local_emojis(folder, group, tmpFolder);
match command {
Command::Local { outputFilepath, folder, originHost, group } => {
zipFilepath = prepare_zip_filepath(outputFilepath)?;
emojis = get_local_emojis(Path::new(folder.as_str()), group, tmpFolder)?;
hostUrl = originHost;
},
Command::Crawl { outputFilepath, host } => {
zipFilepath = prepare_zip_filepath(outputFilepath)?;
emojis = get_host_emojis(&Url::parse(&host).map_err(|_|
report!(EmojiGenError::EmojiFetchFailed)
.attach_printable(format!("Url '{}' is invalid.", host))
)?, tmpFolder)?;
hostUrl = host;
}
}
generate_meta(
originHost,
hostUrl,
emojis,
tmpFolder,
);
zip(tmpFolder.to_str().unwrap(), zipFilepath.to_str().unwrap()).unwrap();
println!("✅ Done! Importable ZIP file under '{}'", zipFilepath.to_str().unwrap());
)?;
zip(tmpFolder, &zipFilepath)?;
drop(tmpFolder);
println!("✅ Done! Importable ZIP file under '{}'", zipFilepath.display());
Ok(())
}
fn crawl(
outputFilepath: String,
host: &Url,
) {
let tmpDir: TempDir = TempDir::new("emoji-gen").unwrap();
let tmpFolder: &Path = tmpDir.path();
let zipFilepath = prepare_zipfile(outputFilepath);
let emojis = get_host_emojis(host, tmpFolder);
generate_meta(
host.to_string(),
emojis,
&tmpFolder,
);
zip(tmpFolder.to_str().unwrap(), zipFilepath.to_str().unwrap()).unwrap();
println!("✅ Done! Importable ZIP file under '{}'", zipFilepath.to_str().unwrap());
drop(&tmpFolder);
}
fn prepare_zipfile (outputFilepath: String) -> PathBuf {
fn prepare_zip_filepath (outputFilepath: String) -> Result<PathBuf, EmojiGenError> {
let mut zipFilepath = PathBuf::from(outputFilepath.as_str());
zipFilepath.set_extension("zip");
assert!(!zipFilepath.exists(), "File '{}' exists. Please choose another name.", zipFilepath.display());
if zipFilepath.exists() {
return Err(report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("File '{}' exists. Please choose another name.", zipFilepath.display())));
}
return zipFilepath;
Ok(zipFilepath)
}
fn get_host_emojis(host: &Url, tmpFolder: &Path) -> Vec<Emoji> {
fn get_host_emojis(host: &Url, tmpFolder: &Path) -> Result<Vec<Emoji>, EmojiGenError> {
println!("Getting all the fine emojis from Url '{}'...", host.as_str());
let hostUrl = &host.join("/api/v1/custom_emojis").unwrap();
let emojis = match reqwest::blocking::get(host.join("/api/v1/custom_emojis").unwrap()) {
let emojis = match reqwest::blocking::get(hostUrl.clone()) {
Ok(response) => match response.json::<Vec<EmojiResponse>>() {
Ok(emojiRes) => {
let emojos = emojiRes;
emojos.iter().map(| res |
Ok(emojos.iter().map(| res |
get_host_emoji_data(
Url::parse(res.url.as_ref().unwrap().as_str().clone()).unwrap(),
res.shortcode.clone().unwrap_or_default(),
res.category.clone().unwrap_or_default(),
tmpFolder,
)
).collect::<Vec<Emoji>>()
).filter_map(|r| r.ok()).collect::<Vec<Emoji>>())
},
Err(_) => {
Err(report!(EmojiGenError::ImageFetchingFailed)
.attach_printable(format!("Could not get emoji list from url '{}'.", hostUrl.as_str())))
},
Err(err) => { println!("{}", err); vec![] },
},
Err(error) => { println!("{}", error); vec![] },
};
Err(_) => {
Err(report!(EmojiGenError::ImageFetchingFailed)
.attach_printable(format!("Could not get response from url '{}'.", hostUrl.as_str())))
},
}?;
return emojis;
Ok(emojis)
}
fn get_host_emoji_data(fileUrl: Url, name: String, category: String, tmpFolder: &Path) -> Emoji {
debug_println!("{}", fileUrl.to_string());
fn get_host_emoji_data(fileUrl: Url, name: String, category: String, tmpFolder: &Path) -> Result<Emoji, EmojiGenError> {
debug!("{}", fileUrl.to_string());
let newFilename = get_image_from_url(&fileUrl, tmpFolder, name.clone());
let newFilename = get_image_from_url(&fileUrl, tmpFolder, name.clone())?;
let data: EmojiData = EmojiData{
name: name,
category: category.to_string(),
aliases: Vec::<String>::new()};
Emoji {
Ok(Emoji {
downloaded: true,
fileName: String::from(newFilename),
fileName: newFilename,
emoji: data
}
})
}
fn get_image_from_url(fileUrl: &Url, tmpFolder: &Path, filename: String) -> String {
fn get_image_from_url(fileUrl: &Url, tmpFolder: &Path, filename: String) -> Result<String, EmojiGenError> {
let img_bytes = &reqwest::blocking::get(fileUrl.clone())
.map_err(|_|
report!(EmojiGenError::ImageFetchingFailed)
.attach_printable(format!("Could not get image file from url '{}'.", fileUrl.as_str()))
)?.bytes().unwrap();
let img_bytes = reqwest::blocking::get(fileUrl.clone()).unwrap().bytes().unwrap();
let extension = getTypename(imghdr::from_bytes(img_bytes.clone()).unwrap_or(Type::Xbm));
let mut tmpFilepath: PathBuf = tmpFolder.join(filename);
tmpFilepath.set_extension(extension);
println!("Creating image file at path '{}'...", tmpFilepath.to_str().unwrap());
match imghdr::from_bytes(img_bytes) {
Some(extension) => tmpFilepath.set_extension(getTypename(extension)),
None => tmpFilepath.set_extension("xxx")
};
println!("Creating image file at path '{}'...", tmpFilepath.display());
let mut imageFile = File::create(tmpFilepath.as_os_str()).unwrap();
imageFile.write_all(&img_bytes).unwrap();
let mut imageFile = File::create(tmpFilepath.as_os_str())
.map_err(|_|
report!(EmojiGenError::ImageFetchingFailed)
.attach_printable(format!("Could not create image file at temporary path '{}'.", tmpFilepath.display()))
)?;
imageFile.write_all(img_bytes)
.map_err(|_|
report!(EmojiGenError::ImageFetchingFailed)
.attach_printable(format!("Could not write image at temporary path '{}'.", tmpFilepath.display()))
)?;
return String::from(tmpFilepath.file_name().unwrap().to_string_lossy());
Ok(String::from(tmpFilepath.file_name().unwrap().to_str().unwrap()))
}
fn get_local_emojis(folder: &Path, group: String, tmpFolder: &Path) -> Vec<Emoji> {
let filesRegex = Regex::new(r"(?:LICENSE|.*\.(?:png|jpe?g|svg))").unwrap();
fn get_local_emojis(folder: &Path, group: String, tmpFolder: &Path) -> Result<Vec<Emoji>, EmojiGenError> {
match folder.canonicalize() {
Ok (f) => {
if !f.is_dir() {
Err(report!(EmojiGenError::EmojiFetchFailed)
.attach_printable(format!("Folder path '{}' is not a directory.", folder.display())))
}
else {
Ok(f)
}
},
Err(_) => {
Err(report!(EmojiGenError::EmojiFetchFailed)
.attach_printable(format!("Folder '{}' does not exist.", folder.display())))
}
}?;
println!("Getting all the fine emojis from folder '{}'...", folder.to_str().unwrap());
println!("Getting all the fine emojis from folder '{}'...", folder.display());
let mut emojis = Vec::<Emoji>::new();
for result in WalkDir::new(folder).into_iter()
.filter(|s| filesRegex.is_match(s.as_ref().unwrap().path().file_name().unwrap().to_str().unwrap())) {
for result in WalkDir::new(folder).into_iter() {
if let Err(_) = result {
continue;
}
@ -289,71 +320,91 @@ fn get_local_emojis(folder: &Path, group: String, tmpFolder: &Path) -> Vec<Emoji
if !file.metadata().unwrap().is_file() {
continue;
}
println!("Checking file '{}'...", file.path().file_name().unwrap().to_str().unwrap());
let subcat = file.path().to_string_lossy()
.replace(file.path().file_name().unwrap().to_string_lossy().to_string().as_str(), "")
.replace(std::path::MAIN_SEPARATOR, "")
.replace(".", "");
debug_println!("{}", file.path().file_name().unwrap().to_str().unwrap());
let filename = file.path().file_name().unwrap();
let emoji = get_local_emoji_data(file, group.clone().into(), subcat, tmpFolder);
if emoji.is_some() {
emojis.push(emoji.unwrap());
println!("Checking file '{}'...", filename.to_string_lossy());
let image = imghdr::from_file(file.path())
.map_err(|_|
report!(EmojiGenError::ImageFetchingFailed)
.attach_printable(format!("Could not get image at path '{}'.", file.path().display()))
)?;
if image.is_none() {
if filename.to_ascii_uppercase() == "LICENSE" || filename.to_ascii_uppercase() == "LICENSE.md" {
get_image_from_path(&file.path(), tmpFolder).map_err(|err| warn!("{}", err)).unwrap();
}
continue;
}
match get_local_emoji_data(file, group.clone().into(), tmpFolder) {
Ok(emoji) => {
emojis.push(emoji);
},
Err(err) => {
// Passing because this is not fatal
warn!("{}", err)
}
}
}
return emojis;
Ok(emojis)
}
fn get_local_emoji_data(file: DirEntry, original_category: Rc<String>, subcategory: String, tmpFolder: &Path) -> Option<Emoji> {
debug_println!("{}", file.path().display());
fn get_local_emoji_data(file: DirEntry, original_category: Rc<String>, tmpFolder: &Path) -> Result<Emoji, EmojiGenError> {
debug!("{}", file.path().display());
let filename = file.file_name();
let image = imghdr::from_file(file.path()).unwrap();
if image.is_none() {
if filename.to_ascii_uppercase() == "LICENSE" || filename.to_ascii_uppercase() == "LICENSE.md" {
get_image_from_path(&file.path(), tmpFolder);
}
let fileName = String::from(file.file_name().to_str().unwrap());
let name = String::from(file.path().file_stem().unwrap().to_str().unwrap()).replace(&[' ', '-'][..], "_");
return None
}
let mut name = file.file_name().to_ascii_lowercase().into_string().unwrap();
name = str::replace(&name, file.path().extension().unwrap().to_string_lossy().to_string().as_str(), "");
name = str::replace(&name, " ", "_");
name = str::replace(&name, "-", "_");
name = str::replace(&name, ".", "");
get_image_from_path(&file.path(), tmpFolder);
get_image_from_path(&file.path(), tmpFolder)?;
let data = EmojiData{
name: name,
category: original_category.to_string() + " - " + subcategory.as_str(),
category: original_category.to_string(),
aliases: Vec::<String>::new()};
Some(Emoji {
Ok(Emoji {
downloaded: true,
fileName: String::from(filename.to_string_lossy()),
fileName,
emoji: data
})
}
fn get_image_from_path(filePath: &Path, tmpFolder: &Path) {
fn get_image_from_path(filePath: &Path, tmpFolder: &Path) -> Result<(), EmojiGenError> {
let filename = filePath.file_name().unwrap();
let img_data = std::fs::read(filePath.as_os_str()).unwrap();
let img_data = std::fs::read(filePath.as_os_str())
.map_err(|_|
report!(EmojiGenError::ImageFetchingFailed)
.attach_printable(format!("Could not read image file at path '{}'.", filePath.display()))
)?;
let img_bytes = img_data.as_slice();
let mut imageFile = File::create(tmpFolder.join(filename)).unwrap();
imageFile.write_all(&img_bytes).unwrap();
let imageFilePath = &tmpFolder.join(filename);
let mut imageFile = File::create(imageFilePath)
.map_err(|_|
report!(EmojiGenError::ImageFetchingFailed)
.attach_printable(format!("Could not create image to temporary path '{}'.", imageFilePath.display()))
)?;
imageFile.write_all(&img_bytes)
.map_err(|_|
report!(EmojiGenError::ImageFetchingFailed)
.attach_printable(format!("Could not write image to temporary path '{}'.", imageFilePath.display()))
)?;
Ok(())
}
fn generate_meta(
host: String,
emojis: Vec<Emoji>,
tmpFolder: &Path
) {
) -> Result<(), EmojiGenError> {
let meta = Meta {
metaVersion: 1,
host: host,
@ -361,72 +412,104 @@ fn generate_meta(
emojis: emojis,
};
let json = serde_json::to_string(&meta).unwrap();
let json = serde_json::to_string(&meta)
.map_err(|_|
report!(EmojiGenError::MetadataGenerationFailed)
.attach_printable(format!("Could not generate metadata for 'meta.json'."))
)?;
let metaFilepath = tmpFolder.join("meta.json");
let metaFilepath: &PathBuf = &tmpFolder.join("meta.json");
println!("Creating file 'meta.json' at path '{}'...", metaFilepath.to_str().unwrap());
let mut file = File::create(metaFilepath).unwrap();
write!(file, "{}", json).unwrap();
}
let mut file = File::create(metaFilepath)
.map_err(|_|
report!(EmojiGenError::MetadataGenerationFailed)
.attach_printable(format!("Could not create file '{}'.", metaFilepath.display()))
)?;
write!(file, "{}", json)
.map_err(|_|
report!(EmojiGenError::MetadataGenerationFailed)
.attach_printable(format!("Could not write metadata to file '{}'.", metaFilepath.display()))
)?;
fn zip(
src_dir: &str,
dst_file: &str,
) -> zip::result::ZipResult<()> {
if !std::path::Path::new(src_dir).is_dir() {
return Err(ZipError::FileNotFound);
}
let path = std::path::Path::new(dst_file);
let file = File::create(path).unwrap();
let walkdir = WalkDir::new(src_dir);
let it = walkdir.into_iter();
zip_dir(&mut it.filter_map(|e| e.ok()), src_dir, file, zip::CompressionMethod::Deflated)?;
Ok(())
}
fn zip_dir<T>(
it: &mut dyn Iterator<Item = DirEntry>,
prefix: &str,
writer: T,
method: zip::CompressionMethod,
) -> zip::result::ZipResult<()>
where
T: Write + Seek,
{
let mut zip = zip::ZipWriter::new(writer);
fn zip(
src_dir: &Path,
dst_file: &Path,
) -> Result<(), EmojiGenError> {
if !std::path::Path::new(src_dir).is_dir() {
return Err(report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("Could not find folder '{}'.", src_dir.display())))
}
let zipFile = &File::create(dst_file).map_err(|_|
report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("Could not create file '{}'.", dst_file.display()))
)?;
let mut zip = zip::ZipWriter::new(zipFile);
let options = FileOptions::default()
.compression_method(method)
.compression_method(zip::CompressionMethod::Deflated)
.unix_permissions(0o755);
let mut buffer = Vec::new();
for entry in it {
for entryRes in WalkDir::new(src_dir).into_iter() {
let entry = entryRes.map_err(|_|
report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("Could get path to a file in folder '{}'.", src_dir.display()))
)?;
let path = entry.path();
let name = path.strip_prefix(std::path::Path::new(prefix)).unwrap();
let name = path.strip_prefix(src_dir).map_err(|_|
report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("Could not strip prefix on file path '{}'.", path.display()))
)?;
// Write file or directory explicitly
// Some unzip tools unzip files with directory paths correctly, some do not!
if path.is_file() {
debug_println!("adding file {:?} as {:?} ...", path, name);
debug!("adding file {:?} as {:?} ...", path, name);
#[allow(deprecated)]
zip.start_file_from_path(name, options)?;
let mut f = File::open(path)?;
zip.start_file_from_path(name, options)
.map_err(|_|
report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("Could not create file '{}' in zip file '{:?}'.", path.display(), zipFile))
)?;
let mut f = File::open(path)
.map_err(|_|
report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("Could not read file '{}'.", path.display()))
)?;
f.read_to_end(&mut buffer)?;
zip.write_all(&*buffer)?;
f.read_to_end(&mut buffer)
.map_err(|_|
report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("Could not read file '{}'.", path.display()))
)?;
zip.write_all(&*buffer)
.map_err(|_|
report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("Could not write data to zip file '{:?}'.", zipFile))
)?;
buffer.clear();
} else if !name.as_os_str().is_empty() {
// Only if not root! Avoids path spec / warning
// and mapname conversion failed error on unzip
debug_println!("adding dir {:?} as {:?} ...", path, name);
debug!("adding dir {:?} as {:?} ...", path, name);
#[allow(deprecated)]
zip.add_directory_from_path(name, options)?;
zip.add_directory_from_path(name, options)
.map_err(|_|
report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("Could not create folder '{}' in zip file '{:?}'.", path.display(), zipFile))
)?;
}
}
zip.finish()?;
zip.finish()
.map_err(|_|
report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("Could not close zip file '{:?}'.", zipFile))
)?;
Ok(())
}