Phuntsok Drak-pa
GitHub
commit
a60f4ac045
6
.gitignore
vendored
6
.gitignore
vendored
@ -1,3 +1,9 @@
|
||||
*~
|
||||
|
||||
gmon\.out
|
||||
/cmake-build-debug/Makefile
|
||||
|
||||
cmake-build-debug/
|
||||
|
||||
\.idea/
|
||||
*.lzw
|
||||
|
||||
@ -8,7 +8,7 @@ set(TGT "projet_lzw")
|
||||
set(${TGT}_VERSION_MAJOR 0)
|
||||
set(${TGT}_VERSION_MINOR 1)
|
||||
|
||||
set(CXX_COVERAGE_COMPILE_FLAGS "-pedantic -Wall -Wextra -Wold-style-cast -Woverloaded-virtual -Wfloat-equal -Wwrite-strings -Wpointer-arith -Wcast-qual -Wcast-align -Wconversion -Wsign-conversion -Wshadow -Weffc++ -Wredundant-decls -Wdouble-promotion -Winit-self -Wswitch-default -Wswitch-enum -Wundef -Winline -Wunused -Wnon-virtual-dtor")
|
||||
set(CXX_COVERAGE_COMPILE_FLAGS "-pedantic -Wall -Wextra -Wold-style-cast -Woverloaded-virtual -Wfloat-equal -Wwrite-strings -Wpointer-arith -Wcast-qual -Wcast-align -Wconversion -Wsign-conversion -Wshadow -Weffc++ -Wredundant-decls -Wdouble-promotion -Winit-self -Wswitch-default -Wswitch-enum -Wundef -Winline -Wunused -Wnon-virtual-dtor -pthread")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "${CXX_COVERAGE_COMPILE_FLAGS} -DDebug -g -pg")
|
||||
set(CMAKE_CXX_FLAGS_RELEASE "${CXX_COVERAGE_COMPILE_FLAGS} -O3")
|
||||
|
||||
|
||||
@ -18,19 +18,19 @@ using dic_t = std::map<std::pair<uint32_t, uint8_t>, uint32_t>;
|
||||
* caractère se référant au dernier caractère de la chaine courante. Si le
|
||||
* numéro de chaine est -1, alors il s’agit du premier caractère de la chaine,
|
||||
* et la valeur renvoyée sera la valeur ASCII du caractère. La fonction renvoie
|
||||
* une paire bool/uint16_t, la valeur booléene indiquant si une nouvelle fut
|
||||
* ajoutée dans le dictionnaire ou non, et le uint16_t indiquant la valeur
|
||||
* une paire bool/uint32_t, la valeur booléene indiquant si une nouvelle fut
|
||||
* ajoutée dans le dictionnaire ou non, et le uint32_t indiquant la valeur
|
||||
* numérique de la chaîne dans le dictionnaire.
|
||||
*
|
||||
* \param dictionary Dictionnaire
|
||||
* \param nr_chaine Numéro de la chaine précédant le caractères \p c dans \p dictionary
|
||||
* \param c Caractère suivant la chaine de caractères \p nr_chaine
|
||||
* \return std::pair<bool, uint16_t>
|
||||
* \return std::pair<bool, uint32_t>
|
||||
*/
|
||||
const std::pair<bool, uint32_t>
|
||||
dico(std::map<std::pair<uint32_t, uint8_t>, uint32_t> &t_dictionary,
|
||||
uint32_t t_nr_chaine, uint8_t t_c) {
|
||||
if (t_nr_chaine == 0xFFFFFFFF)
|
||||
if (t_nr_chaine == 0xFFFF)
|
||||
return std::make_pair(true, t_c);
|
||||
auto &e = t_dictionary[std::make_pair(t_nr_chaine, t_c)];
|
||||
return (e) ? std::make_pair(true, e)
|
||||
|
||||
@ -11,8 +11,7 @@
|
||||
|
||||
/// \brief Recherche ou ajout de chaine dans le dictionnaire
|
||||
const std::pair<bool, std::uint32_t>
|
||||
dico(std::map<std::pair<std::uint32_t, std::uint8_t>, std::uint32_t>
|
||||
&t_dictionary,
|
||||
std::uint32_t t_nr_chaine, std::uint8_t t_c);
|
||||
dico(std::map<std::pair<std::uint32_t, std::uint8_t>, std::uint32_t> &,
|
||||
uint32_t, uint8_t);
|
||||
|
||||
#endif /* LZW_SRC_COMMON_H_ */
|
||||
|
||||
154
src/compress.cc
154
src/compress.cc
@ -5,58 +5,67 @@
|
||||
|
||||
#include "compress.hh"
|
||||
#include "utf8.hh"
|
||||
#include <cassert>
|
||||
#include <cstdlib>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <thread>
|
||||
|
||||
#ifdef Debug
|
||||
constexpr bool debug_mode = true;
|
||||
#else
|
||||
constexpr bool debug_mode = false;
|
||||
#endif
|
||||
|
||||
using dict_t = std::map<std::pair<uint32_t, uint8_t>, uint32_t>;
|
||||
using ustring = std::basic_string<uint8_t>; // chaine non encodée
|
||||
using uvec = std::vector<uint32_t>; // chaine encodée
|
||||
using ustring = std::basic_string<uint8_t>; // chaîne non encodée
|
||||
using uvec = std::vector<uint32_t>; // chaîne encodée
|
||||
using std::printf;
|
||||
|
||||
void join_and_write(
|
||||
std::vector<std::pair<std::unique_ptr<std::thread>, uvec>> &t_threads,
|
||||
FILE *t_out) {
|
||||
for (auto &elem : t_threads) {
|
||||
(*elem.first).join();
|
||||
}
|
||||
for (auto &elem : t_threads) {
|
||||
for (const auto c : elem.second) {
|
||||
write_utf8(t_out, c);
|
||||
}
|
||||
}
|
||||
t_threads.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* La chaine de caractère \p text est lue caractère par caractère, et est et
|
||||
* La chaîne de caractère \p t_text est lue caractère par caractère, et est et
|
||||
* selon la valeur de retour de la fonction \ref dico (permettant dans le même
|
||||
* temps la création du dictionnaire), on rajoute un mot ou pas dans le vecteur
|
||||
* de caractères UTF-8 représentant des mots de chars compressés. La fonction
|
||||
* renvoie ledit vecteur de uint32_t.
|
||||
* renvoie ledit vecteur de uint32_t via le paramètre \p t_res.
|
||||
*
|
||||
* \param t_text Chaine de caractères uint8_t représentant le fichier d'entrée
|
||||
* \param t_dictionary Dictionnaire de compression
|
||||
* \return std::vector<uint16_t>
|
||||
* \param[in] t_text Chaîne de caractères uint8_t représentant le fichier
|
||||
* d'entrée \param[out] t_res Chaîne de caractères de sortie
|
||||
*/
|
||||
const uvec lzw_compress(const ustring &t_text, dict_t &t_dictionary) {
|
||||
void lzw_compress(const std::vector<char> &t_text, uvec &t_res) {
|
||||
dict_t dictionary{};
|
||||
std::puts("Compressing...");
|
||||
uvec res{};
|
||||
uint32_t w = 0xFFFFFFFF;
|
||||
uint32_t len = 0;
|
||||
uint32_t w = 0xFFFF;
|
||||
|
||||
constexpr size_t DICT_MAX = 7936; /* 12 bits */
|
||||
|
||||
#ifdef Debug
|
||||
size_t progress = 0;
|
||||
#endif
|
||||
|
||||
for (const auto &c : t_text) {
|
||||
++len;
|
||||
|
||||
#ifdef Debug
|
||||
printf("\rprogress: %zu / %zu", ++progress, t_text.size());
|
||||
#endif
|
||||
|
||||
if (/* len > LENGTH_MAX || */ t_dictionary.size() >= DICT_MAX) {
|
||||
res.push_back(static_cast<uint32_t>(w));
|
||||
w = c;
|
||||
len = 0;
|
||||
} else if (const auto &[exists, pos] = dico(t_dictionary, w, c); exists) {
|
||||
if (dictionary.size() >= DICT_MAX) {
|
||||
t_res.push_back(static_cast<uint32_t>(w));
|
||||
w = static_cast<uint32_t>(c);
|
||||
} else if (const auto &[exists, pos] =
|
||||
dico(dictionary, w, static_cast<std::uint8_t>(c));
|
||||
exists) {
|
||||
w = pos;
|
||||
} else {
|
||||
res.push_back(static_cast<uint32_t>(w));
|
||||
w = c;
|
||||
len = 0;
|
||||
t_res.push_back(static_cast<uint32_t>(w));
|
||||
w = static_cast<std::uint8_t>(c);
|
||||
}
|
||||
}
|
||||
printf("\n");
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -73,41 +82,66 @@ const uvec lzw_compress(const ustring &t_text, dict_t &t_dictionary) {
|
||||
void compress(const std::string &t_in_file, const char *t_out_file) {
|
||||
// Fichier d’entrée
|
||||
std::ifstream input_file{t_in_file};
|
||||
if (!input_file.is_open()) {
|
||||
std::cerr << "Error at " << __FILE__ << ":" << __LINE__ - 2
|
||||
<< ": could not open output file \"" << t_in_file
|
||||
<< "\". Aborting...\n";
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// Fichier de sortie
|
||||
FILE *out = (t_out_file)
|
||||
? fopen(t_out_file, "wb")
|
||||
: fopen(std::string{t_out_file, ".lzw"}.c_str(), "wb");
|
||||
const char *filename = (t_out_file) ? t_out_file : "output.lzw";
|
||||
FILE *out = fopen(filename, "wb");
|
||||
if (!out) {
|
||||
std::cerr << "Error at " << __FILE__ << ":" << __LINE__ - 4
|
||||
<< ": could not open output file \"" << filename
|
||||
<< "\". Aborting...\n";
|
||||
input_file.close();
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// thread pool
|
||||
std::vector<std::pair<std::unique_ptr<std::thread>, uvec>> threads{};
|
||||
|
||||
// char chunk[32768];
|
||||
std::vector<char> chunk(32768, 0);
|
||||
while (input_file.read(chunk.data(),
|
||||
static_cast<std::streamsize>(chunk.size()))) {
|
||||
threads.emplace_back(nullptr, uvec{});
|
||||
threads.back().first = std::unique_ptr<std::thread>(
|
||||
new std::thread{lzw_compress, chunk, ref(threads.back().second)});
|
||||
assert(threads.back().first);
|
||||
if (threads.size() >= 8) {
|
||||
join_and_write(threads, out);
|
||||
}
|
||||
}
|
||||
|
||||
if (!threads.empty()) {
|
||||
join_and_write(threads, out);
|
||||
}
|
||||
|
||||
if (input_file.tellg() != std::ios::end) {
|
||||
std::puts("Leftovers, compressing...");
|
||||
{
|
||||
const auto prev_pos = input_file.tellg();
|
||||
input_file.seekg(0, std::ios::end);
|
||||
// string contenant le fichier d’entrée
|
||||
ustring str(static_cast<unsigned long>(input_file.tellg()),
|
||||
static_cast<unsigned char>(0));
|
||||
input_file.seekg(0, std::ios::beg);
|
||||
|
||||
// assignation du contenu du fichier à str
|
||||
str.assign((std::istreambuf_iterator<char>(input_file)),
|
||||
std::istreambuf_iterator<char>());
|
||||
|
||||
printf("Size of input file: %zu\n", str.size());
|
||||
|
||||
dict_t dictionary{};
|
||||
|
||||
const auto comp_str{lzw_compress(str, dictionary)};
|
||||
|
||||
printf("\n############################################\n");
|
||||
printf(" Compressed!\n");
|
||||
printf("############################################\n\n");
|
||||
printf("Size of compressed string: %zu\n", comp_str.size());
|
||||
printf("Size of dictionary: %zu\n", dictionary.size());
|
||||
printf("Compression ratio: %.10f\n",
|
||||
static_cast<double>(str.size() / comp_str.size()));
|
||||
|
||||
for(const auto c : comp_str)
|
||||
chunk.reserve(static_cast<size_t>(input_file.tellg() - prev_pos));
|
||||
input_file.seekg(prev_pos, std::ios::beg);
|
||||
std::istreambuf_iterator<char> itr(input_file);
|
||||
for (std::streamoff i = 0; i < prev_pos; ++i, ++itr)
|
||||
;
|
||||
chunk.assign((itr), std::istreambuf_iterator<char>());
|
||||
}
|
||||
uvec ret{};
|
||||
lzw_compress(chunk, ret);
|
||||
for (const auto c : ret) {
|
||||
if constexpr (debug_mode) {
|
||||
printf("%c\t", c);
|
||||
}
|
||||
write_utf8(out, c);
|
||||
}
|
||||
}
|
||||
|
||||
fclose(out);
|
||||
input_file.close();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
@ -8,14 +8,12 @@
|
||||
|
||||
#include "common.hh"
|
||||
#include <vector>
|
||||
#include <iostream>
|
||||
|
||||
/// \brief Compression d'une chaine de caractères
|
||||
const std::vector<std::uint32_t>
|
||||
lzw_compress(const std::basic_string<uint8_t> &t_text,
|
||||
std::map<std::pair<std::uint32_t, std::uint8_t>, std::uint32_t>
|
||||
&t_dictionary);
|
||||
void lzw_compress(const std::vector<char> &, std::vector<std::uint32_t> &);
|
||||
|
||||
/// \brief Wrapper de \ref lzw_compress
|
||||
void compress(const std::string &t_in_file, const char *t_out_file);
|
||||
void compress(const std::string &, const char *);
|
||||
|
||||
#endif /* LZW_SRC_COMPRESS_H_ */
|
||||
|
||||
56
src/main.cc
56
src/main.cc
@ -6,6 +6,12 @@
|
||||
*
|
||||
*/
|
||||
|
||||
#ifdef Debug
|
||||
constexpr bool debug_mode = true;
|
||||
#else
|
||||
constexpr bool debug_mode = false;
|
||||
#endif
|
||||
|
||||
#include "compress.hh"
|
||||
#include "getopt.h"
|
||||
|
||||
@ -48,16 +54,17 @@ void help() {
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
#ifdef Debug
|
||||
|
||||
if constexpr (debug_mode) {
|
||||
for (int i = 0; i < argc; ++i)
|
||||
printf("argv[%d] = %s\n", i, argv[i]);
|
||||
#endif
|
||||
}
|
||||
|
||||
std::string input_path{};
|
||||
std::string output_path{};
|
||||
bool compressing = true;
|
||||
|
||||
while (1) {
|
||||
while (true) {
|
||||
int option_index = 0;
|
||||
static struct option long_options[] = {
|
||||
{"help", no_argument, nullptr, 'h'},
|
||||
@ -65,67 +72,68 @@ int main(int argc, char *argv[]) {
|
||||
{"output", required_argument, nullptr, 'o'},
|
||||
{"compress", no_argument, nullptr, 'c'},
|
||||
{"uncompress", no_argument, nullptr, 'u'},
|
||||
{0, 0, 0, 0}};
|
||||
{nullptr, 0, nullptr, 0}};
|
||||
int c = getopt_long(argc, argv, "hi:o:cu", long_options, &option_index);
|
||||
if (c == -1)
|
||||
break;
|
||||
switch (c) {
|
||||
case 0: {
|
||||
#ifdef Debug
|
||||
if constexpr (debug_mode) {
|
||||
printf("\noption %s", long_options[option_index].name);
|
||||
if (optarg) {
|
||||
printf(" with arg %s\n", optarg);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'h': {
|
||||
#ifdef Debug
|
||||
if constexpr (debug_mode) {
|
||||
printf("From main - option --help passed\n");
|
||||
#endif
|
||||
}
|
||||
help();
|
||||
return 0;
|
||||
}
|
||||
case 'i': {
|
||||
#ifdef Debug
|
||||
if constexpr (debug_mode) {
|
||||
printf("From main - option --input with value '%s'\n", optarg);
|
||||
#endif
|
||||
}
|
||||
input_path = optarg;
|
||||
break;
|
||||
}
|
||||
case 'o': {
|
||||
#ifdef Debug
|
||||
if constexpr (debug_mode) {
|
||||
printf("From main - option --output with value '%s'\n", optarg);
|
||||
#endif
|
||||
}
|
||||
output_path = optarg;
|
||||
break;
|
||||
}
|
||||
case 'c': {
|
||||
#ifdef Debug
|
||||
if constexpr (debug_mode) {
|
||||
printf("From main - option --compress\n");
|
||||
#endif
|
||||
}
|
||||
compressing = true;
|
||||
break;
|
||||
}
|
||||
case 'u': {
|
||||
#ifdef Debug
|
||||
if constexpr (debug_mode) {
|
||||
printf("From main - option --uncompress\n");
|
||||
#endif
|
||||
}
|
||||
compressing = false;
|
||||
break;
|
||||
}
|
||||
case '?': {
|
||||
case '?':
|
||||
default: {
|
||||
puts("Error: unknown parameter.");
|
||||
#ifdef Debug
|
||||
if constexpr (debug_mode) {
|
||||
printf("From main - option -?\n");
|
||||
#endif
|
||||
}
|
||||
help();
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (input_path == "") {
|
||||
if (input_path.empty()) {
|
||||
puts("Error: no input file specified");
|
||||
return 2;
|
||||
}
|
||||
@ -140,7 +148,15 @@ int main(int argc, char *argv[]) {
|
||||
- bit-packing, limiter la taille du dictionnaire pour un certain nombre de
|
||||
bits.
|
||||
*/
|
||||
if constexpr (debug_mode) {
|
||||
puts("Beginning compression");
|
||||
}
|
||||
if(output_path.empty()) {
|
||||
compress(input_path, nullptr);
|
||||
} else {
|
||||
compress(input_path, output_path.c_str());
|
||||
}
|
||||
// compress(input_path, output_path.c_str());
|
||||
} else {
|
||||
puts("Not yet implemented :(");
|
||||
/*
|
||||
|
||||
Loading…
Reference in New Issue
Block a user