统计单词出现频率及排序从单机到多机合作图文示例

def 发表于 2017-3-3 09:49:09

技术博客http://www.cnblogs.com/itdef/http://blog.csdn.net/stecdeng 技术交流群群号码：324164944 欢迎c c++ windows驱动爱好者服务器程序员沟通交流
本文是学习多线程服务端编程的练习

书籍作者陈硕的博客也有提到这个题目

http://blog.csdn.net/solstice/article/details/8497475

第一个层次很简单单机一个小文件读进来进行处理然后对每个单词进行统计排序记录每个单词出现频率#include <algorithm>
#include <iostream>
#include <unordered_map>
#include <vector>
#include <sstream>
#include <fstream>

using namespace std;

std::unordered_map<std::string, int> mapStringFrequent;

void SplitStrig(std::string& line, std::vector<std::string>& vecString) {
typedef std::string::size_type strSize;
strSize i = 0;
while (i != line.size()) {
   while (i != line.size() && isspace(line))
         ++i;
   strSize j = i;
   while (j != line.size() && !isspace(line))
         ++j;
   if (i != j) {
         vecString.push_back(line.substr(i, j - i));
         i = j;
   }
}
}

bool CheckWord(std::string s) {
typedef std::string::size_type strSize;
strSize i = 0;
while (i != s.size()) {
   if (isalpha(s)) {
         i++;
   }
   else {
         return false;
   }
}
return true;
}

void VecInputMap(std::vector<std::string>& vecString, std::unordered_map<std::string, int>& mapStringFrequent) {
for (auto it : vecString) {
   if (CheckWord(it)) {
         transform(it.begin(), it.end(), it.begin(), ::tolower);
         mapStringFrequent++;
   }
}
}

bool ReadFile(const string& fileName) {
bool ret = false;
std::ifstream in(fileName);
if (in.bad() || !in) {
   return ret;
}
std::string line;
std::vector<std::string> vecString;
try {
   while (getline(in, line))
   {
         //std::cout << line <<std::endl;
         SplitStrig(line, vecString);
   }

   VecInputMap(vecString, mapStringFrequent);
}
catch (std::exception& e) {
   std::cout << line << std::endl;
   in.close();
   std::cerr << e.what() << std::endl;
   return ret;
}

in.close();
ret = true;
return ret;
}

void WriteToFile(const string& fileName, std::vector<std::pair<int, std::string>>& vecWordFreq) {
std::ofstream out(fileName);
if (out.bad() || !out) {
   return;
}

for (auto& it : vecWordFreq) {
   out << it.second << '\t' << it.first << std::endl;
}
out.close();

}
//============================================================

int main()
{
bool ret = ReadFile("1.txt");
if (!ret)
   return -1;

//ret = ReadFile("2.txt");
//if (!ret)
//return -1;
//ret = ReadFile("3.txt");
//if (!ret)
//return -1;
//ret = ReadFile("4.txt");
//if (!ret)
//return -1;

//ret = ReadFile("5.txt");
//if (!ret)
//return -1;

//ret = ReadFile("6.txt");
//if (!ret)
//return -1;

//ret = ReadFile("7.txt");
//if (!ret)
//return -1;

//ret = ReadFile("8.txt");
//if (!ret)
//return -1;

//ret = ReadFile("9.txt");
//if (!ret)
//return -1;

//ret = ReadFile("10.txt");
//if (!ret)
//return -1;

std::vector<std::pair<int, std::string>> freq;
freq.reserve(mapStringFrequent.size());

for (auto& it : mapStringFrequent){
   freq.push_back(make_pair(it.second, it.first));
}

std::sort(freq.begin(), freq.end(), [](const std::pair<int, std::string>& lhs,// const auto& lhs in C++14
         const std::pair<int, std::string>& rhs) {
   return lhs.first > rhs.first;
});

//for (auto it : freq) {
//std::cout << it.first << '\t' << it.second << '\n';
//}

WriteToFile("freqResult.txt", freq);
return 0;
}第二个层次就是文件较大单词量较多如果一次性读入并使用vector容器存放以及使用hash容器进行统计频率会出现内存等资源不足现象

那么就依次读取进内存将解析的单词存放进vector中，当vector中的容量到达一个阈值将vector中的单词放进map容器中统计单词出现频率 vector容器清空

但是map容器有多个各个单词存放进那个map容器根据hash函数决定

当map容器的容量达到阈值后写入文件 map容器清空

这样我们就得到多个记录单词出现频率的文本供后继步骤分析合并但是一个单词的出现频率肯定只出现在一个记录文本中(因为相同的单词具有相同的hash值对应同一个记录文本)
如图
#include <algorithm>
#include <iostream>
#include <unordered_map>
#include <vector>
#include <sstream>
#include <fstream>

using namespace std;

#define BUCKET_NUM10
std::unordered_map<std::string, int> mapStringFrequent;
std::ofstream out;

#define MAX_WORD_SIZE 1024*1024

void SplitStrig(std::string& line, std::vector<std::string>& vecString) {
typedef std::string::size_type strSize;
strSize i = 0;
while (i != line.size()) {
   while (i != line.size() && isspace(line))
         ++i;
   strSize j = i;
   while (j != line.size() && !isspace(line))
         ++j;
   if (i != j) {
         vecString.push_back(line.substr(i, j - i));
         i = j;
   }
}
}

bool CheckWord(std::string s) {
typedef std::string::size_type strSize;
strSize i = 0;
while (i != s.size()) {
   if (isalpha(s)) {
         i++;
   }
   else {
         return false;
   }
}
return true;
}

void VecInputMap(std::vector<std::string>& vecString, std::unordered_map<std::string, int> mapStringFrequent[]) {
for (auto it : vecString) {
   if (CheckWord(it)) {
         transform(it.begin(), it.end(), it.begin(), ::tolower);
         int idx = std::hash<string>()(it) % BUCKET_NUM;
         mapStringFrequent++;
         if (mapStringFrequent.size() >= MAX_WORD_SIZE) {
            for (auto& it : mapStringFrequent) {
               out << it.second << '\t' << it.first << std::endl;
            }
            mapStringFrequent.clear();
         }
   }
}
}

bool ReadFile(const string& fileName) {
bool ret = false;
std::ifstream in(fileName);
if (in.bad() || !in) {
   return ret;
}
std::string line;
std::vector<std::string> vecString;

try {
   while (getline(in, line))
   {
         SplitStrig(line, vecString);
         //std::cout << line << std::endl;
         if (vecString.size() >= MAX_WORD_SIZE) {
            //std::cout << "too many words" << std::endl;
            VecInputMap(vecString, mapStringFrequent);
            vecString.clear();
         }
   }

   for (int i = 0; i < BUCKET_NUM; i++) {
         for (auto& it : mapStringFrequent) {
            out << it.second << '\t' << it.first << std::endl;
         }
         mapStringFrequent.clear();
   }

}
catch (std::exception& e) {
   in.close();
   std::cerr << e.what() << std::endl;
   return ret;
}

std::cout << vecString.size() << std::endl;
ret = true;
return ret;
}

int main()
{
std::string fileName = "0bucket.txt";
for (int i = 0; i < BUCKET_NUM; i++) {
   out.close();
   out.open(fileName);
   fileName++;
}

ReadFile("LargeFile.txt");

return 0;
}运行效果如图

运行前

运行后

0xAA55 发表于 2017-3-8 22:16:00

我已编辑代码部分的帖子，用了代码上色。
discuz一个大坑就是和是斜体，但C系代码中经常会出现用引用数组元素的语法，导致莫名其妙的斜体……

页: [1]

技术宅的结界's Archiver

统计单词出现频率及排序 从单机到多机合作 图文示例

统计单词出现频率及排序从单机到多机合作图文示例