phundrak
/
lzw-assignment
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

parallel compression kinda working, but ultimately segfault

This commit is contained in:
Lucien Cartier-Tilet 2018-04-26 11:54:02 +02:00
parent b1f3b5839f
commit a4f722f6fb
6 changed files with 142 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -1,3 +1,8 @@
*~
gmon\.out
/cmake-build-debug/Makefile
cmake-build-debug/
\.idea/

12
src/common.cc
View File

@ -6,8 +6,8 @@
#include "common.hh"
using uint8_t = std::uint8_t;
using uint32_t = std::uint32_t;
using dic_t = std::map<std::pair<uint32_t, uint8_t>, uint32_t>;
using uint16_t = std::uint16_t;
using dic_t = std::map<std::pair<uint16_t, uint8_t>, uint16_t>;
/**
* Cette fonction a pour double usage la recherche d’une chaine de caractères
@ -27,10 +27,10 @@ using dic_t = std::map<std::pair<uint32_t, uint8_t>, uint32_t>;
* \param c Caractère suivant la chaine de caractères \p nr_chaine
* \return std::pair<bool, uint16_t>
*/
const std::pair<bool, uint32_t>
dico(std::map<std::pair<uint32_t, uint8_t>, uint32_t> &t_dictionary,
uint32_t t_nr_chaine, uint8_t t_c) {
if (t_nr_chaine == 0xFFFFFFFF)
const std::pair<bool, uint16_t>
dico(std::map<std::pair<uint16_t, uint8_t>, uint16_t> &t_dictionary,
uint16_t t_nr_chaine, uint8_t t_c) {
if (t_nr_chaine == 0xFFFF)
return std::make_pair(true, t_c);
auto &e = t_dictionary[std::make_pair(t_nr_chaine, t_c)];
return (e) ? std::make_pair(true, e)

7
src/common.hh
View File

@ -10,9 +10,8 @@
#include <map>
/// \brief Recherche ou ajout de chaine dans le dictionnaire
const std::pair<bool, std::uint32_t>
dico(std::map<std::pair<std::uint32_t, std::uint8_t>, std::uint32_t>
&t_dictionary,
std::uint32_t t_nr_chaine, std::uint8_t t_c);
const std::pair<bool, std::uint16_t>
dico(std::map<std::pair<std::uint16_t, std::uint8_t>, std::uint16_t> &,
uint16_t, uint8_t);
#endif /* LZW_SRC_COMMON_H_ */

123
src/compress.cc
View File

@ -3,14 +3,22 @@
* \brief Implementation of compression
*/
#ifdef Debug
constexpr bool debug_mode = true;
#else
constexpr bool debug_mode = false;
#endif
#include "compress.hh"
#include "utf8.hh"
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <thread>
using dict_t = std::map<std::pair<uint32_t, uint8_t>, uint32_t>;
using dict_t = std::map<std::pair<uint16_t, uint8_t>, uint16_t>;
using ustring = std::basic_string<uint8_t>; // chaine non encodée
using uvec = std::vector<uint32_t>; // chaine encodée
using uvec = std::vector<uint16_t>; // chaine encodée
using std::printf;
/**
@ -24,39 +32,40 @@ using std::printf;
* \param t_dictionary Dictionnaire de compression
* \return std::vector<uint16_t>
*/
const uvec lzw_compress(const ustring &t_text, dict_t &t_dictionary) {
void lzw_compress(const std::vector<char> &t_text, uvec &t_res) {
dict_t dictionary{};
std::puts("Compressing...");
uvec res{};
uint32_t w = 0xFFFFFFFF;
uint32_t len = 0;
// uvec res{};
uint16_t w = 0xFFFF;
uint16_t len = 0;
constexpr size_t DICT_MAX = 7936; /* 12 bits */
#ifdef Debug
size_t progress = 0;
#endif
for (const auto &c : t_text) {
++len;
#ifdef Debug
printf("\rprogress: %zu / %zu", ++progress, t_text.size());
#endif
if constexpr (debug_mode) {
printf("\rprogress: %zu / %zu", ++progress, t_text.size());
}
if (/* len > LENGTH_MAX || */ t_dictionary.size() >= DICT_MAX) {
res.push_back(static_cast<uint32_t>(w));
w = c;
if (/* len > LENGTH_MAX || */ dictionary.size() >= DICT_MAX) {
t_res.push_back(static_cast<uint16_t>(w));
w = static_cast<uint16_t>(c);
len = 0;
} else if (const auto &[exists, pos] = dico(t_dictionary, w, c); exists) {
} else if (const auto &[exists, pos] =
dico(dictionary, w, static_cast<std::uint8_t>(c));
exists) {
w = pos;
} else {
res.push_back(static_cast<uint32_t>(w));
w = c;
t_res.push_back(static_cast<uint16_t>(w));
w = static_cast<std::uint8_t>(c);
len = 0;
}
}
printf("\n");
return res;
// return res;
}
/**
@ -73,41 +82,69 @@ const uvec lzw_compress(const ustring &t_text, dict_t &t_dictionary) {
void compress(const std::string &t_in_file, const char *t_out_file) {
// Fichier d’entrée
std::ifstream input_file{t_in_file};
if(!input_file.is_open()) {
std::cerr << "Error at " << __FILE__ << ":" << __LINE__ - 2
<< ": could not open output file \"" << t_in_file << "\". Aborting...\n";
exit(1);
}
// Fichier de sortie
FILE *out = (t_out_file)
? fopen(t_out_file, "wb")
: fopen(std::string{t_out_file, ".lzw"}.c_str(), "wb");
const char *filename =
(t_out_file) ? t_out_file : "output.lzw";
FILE *out = fopen(filename, "wb");
if(!out) {
std::cerr << "Error at " << __FILE__ << ":" << __LINE__ - 4
<< ": could not open output file \"" << filename << "\". Aborting...\n";
input_file.close();
exit(1);
}
input_file.seekg(0, std::ios::end);
// string contenant le fichier d’entrée
ustring str(static_cast<unsigned long>(input_file.tellg()),
static_cast<unsigned char>(0));
input_file.seekg(0, std::ios::beg);
// input_file.seekg(0, std::ios::end);
// // string contenant le fichier d’entrée
// ustring str(static_cast<unsigned long>(input_file.tellg()),
// static_cast<unsigned char>(0));
// input_file.seekg(0, std::ios::beg);
// assignation du contenu du fichier à str
str.assign((std::istreambuf_iterator<char>(input_file)),
std::istreambuf_iterator<char>());
// // assignation du contenu du fichier à str
// str.assign((std::istreambuf_iterator<char>(input_file)),
// std::istreambuf_iterator<char>());
printf("Size of input file: %zu\n", str.size());
// printf("Size of input file: %zu\n", str.size());
dict_t dictionary{};
// dict_t dictionary{};
const auto comp_str{lzw_compress(str, dictionary)};
// const auto comp_str{lzw_compress(str, dictionary)};
printf("\n############################################\n");
printf(" Compressed!\n");
printf("############################################\n\n");
printf("Size of compressed string: %zu\n", comp_str.size());
printf("Size of dictionary: %zu\n", dictionary.size());
printf("Compression ratio: %.10f\n",
static_cast<double>(str.size() / comp_str.size()));
// thread pool
std::vector<std::pair<std::thread, uvec>> threads{};
for(const auto c : comp_str)
write_utf8(out, c);
// char chunk[32768];
std::vector<char> chunk{};
chunk.reserve(32768);
while (input_file.read(chunk.data(), 32768)) {
threads.push_back(std::make_pair(std::thread{}, uvec{}));
threads.back().first =
std::thread{lzw_compress, chunk, ref(threads.back().second)};
if (threads.size() >= 8) {
for (auto &elem : threads) {
elem.first.join();
}
for (auto &elem : threads) {
for (const auto c : elem.second) {
write_utf8(out, c);
}
}
threads.clear();
}
}
if(input_file.tellg() != std::ios::end) {
std::puts("Leftovers...");
}
// for (const auto c : comp_str)
// write_utf8(out, c);
fclose(out);
input_file.close();
return;
}

7
src/compress.hh
View File

@ -8,12 +8,11 @@
#include "common.hh"
#include <vector>
#include <iostream>
/// \brief Compression d'une chaine de caractères
const std::vector<std::uint32_t>
lzw_compress(const std::basic_string<uint8_t> &t_text,
std::map<std::pair<std::uint32_t, std::uint8_t>, std::uint32_t>
&t_dictionary);
void lzw_compress(const std::vector<char> &t_text,
std::vector<std::uint16_t> &t_rec);
/// \brief Wrapper de \ref lzw_compress
void compress(const std::string &t_in_file, const char *t_out_file);

74
src/main.cc
View File

@ -6,6 +6,12 @@
*
*/
#ifdef Debug
constexpr bool debug_mode = true;
#else
constexpr bool debug_mode = false;
#endif
#include "compress.hh"
#include "getopt.h"
@ -48,10 +54,11 @@ void help() {
}
int main(int argc, char *argv[]) {
#ifdef Debug
for (int i = 0; i < argc; ++i)
printf("argv[%d] = %s\n", i, argv[i]);
#endif
if constexpr (debug_mode) {
for (int i = 0; i < argc; ++i)
printf("argv[%d] = %s\n", i, argv[i]);
}
std::string input_path{};
std::string output_path{};
@ -71,54 +78,55 @@ int main(int argc, char *argv[]) {
break;
switch (c) {
case 0: {
#ifdef Debug
printf("\noption %s", long_options[option_index].name);
if (optarg) {
printf(" with arg %s\n", optarg);
if constexpr (debug_mode) {
printf("\noption %s", long_options[option_index].name);
if (optarg) {
printf(" with arg %s\n", optarg);
}
}
#endif
break;
}
case 'h': {
#ifdef Debug
printf("From main - option --help passed\n");
#endif
if constexpr (debug_mode) {
printf("From main - option --help passed\n");
}
help();
return 0;
}
case 'i': {
#ifdef Debug
printf("From main - option --input with value '%s'\n", optarg);
#endif
if constexpr (debug_mode) {
printf("From main - option --input with value '%s'\n", optarg);
}
input_path = optarg;
break;
}
case 'o': {
#ifdef Debug
printf("From main - option --output with value '%s'\n", optarg);
#endif
if constexpr (debug_mode) {
printf("From main - option --output with value '%s'\n", optarg);
}
output_path = optarg;
break;
}
case 'c': {
#ifdef Debug
printf("From main - option --compress\n");
#endif
if constexpr (debug_mode) {
printf("From main - option --compress\n");
}
compressing = true;
break;
}
case 'u': {
#ifdef Debug
printf("From main - option --uncompress\n");
#endif
if constexpr (debug_mode) {
printf("From main - option --uncompress\n");
}
compressing = false;
break;
}
case '?': {
case '?':
default: {
puts("Error: unknown parameter.");
#ifdef Debug
printf("From main - option -?\n");
#endif
if constexpr (debug_mode) {
printf("From main - option -?\n");
}
help();
return 1;
}
@ -140,7 +148,15 @@ int main(int argc, char *argv[]) {
- bit-packing, limiter la taille du dictionnaire pour un certain nombre de
bits.
*/
compress(input_path, output_path.c_str());
if constexpr (debug_mode) {
puts("Beginning compression");
}
if(output_path.empty()) {
compress(input_path, nullptr);
} else {
compress(input_path, output_path.c_str());
}
// compress(input_path, output_path.c_str());
} else {
puts("Not yet implemented :(");
/*