利用文本挖掘技术来找出网络中的“小鲜词”
发布时间:2021-01-17 22:16:03 所属栏目:大数据 来源:网络整理
导读:副标题#e# 开始之前,先看一下从人人网中发现的90后用户爱用的词 是不是很好玩,哈哈。写这篇文章就是让你简单的自动的从文本中找出新的词,这样就知道现在的年轻人喜欢什么了(对于博主这种上了年纪的人来说,真的是很有用,呜呜) 项目结构 当然,text.da
|
这几个类用于给词创建索引,方便从词典中找出 CnPreviewTextIndexer.javapackage grid.text.index;
import grid.common.TextUtils;
import java.util.HashMap;
import java.util.Map;
import java.util.Vector;
public class CnPreviewTextIndexer implements TextIndexer {
private final static int CN_LETTER_COUNT = 5021;
private String document;
private Map<Character,Vector<Integer>> posMap;
public CnPreviewTextIndexer(String document) {
this.document = document;
init();
}
private void init() {
final int len = document.length();
final int supposedMinCount = 1 + (int) Math.log(len / CN_LETTER_COUNT
+ 1);
char c;
Vector<Integer> posVector;
posMap = new HashMap<Character,Vector<Integer>>(CN_LETTER_COUNT);
for (int i = 0; i < len; i++) {
c = document.charAt(i);
if (!TextUtils.isCnLetter(c)) {
continue;
}
posVector = posMap.get(c);
if (null == posVector) {
posVector = new Vector<Integer>(supposedMinCount);
posMap.put(c,posVector);
}
posVector.add(i);
}
}
@Override
public int count(String text) {
if (TextUtils.isBlank(text)) {
return 0;
}
Vector<Integer> vector = posMap.get(text.charAt(0));
if (null == vector) {
return 0;
}
if (1 == text.length()) {
return vector.size();
}
final int size = vector.size();
int count = 0;
for (int i = 0; i < size; i++) {
if (TextUtils.match(document,vector.get(i),text)) {
count++;
}
}
return count;
}
@Override
public Pos find(Pos pos) {
String text = pos.getTarget();
pos.setFound(false);
if (TextUtils.isBlank(text)) {
return pos;
}
Vector<Integer> vector = posMap.get(text.charAt(0));
if (null == vector) {
return pos;
}
final int arraySize = vector.size();
final int arrayIndex = pos.arrayIndex + 1;
for (int i = arrayIndex; i < arraySize; i++) {
if (TextUtils.match(document,text)) {
pos.setFound(true);
pos.setPos(vector.get(i));
pos.arrayIndex = i;
break;
}
}
return pos;
}
@Override
public int len() {
return document.length();
}
@Override
public String sub(int off,int len) {
if (off < 0 || off + len >= document.length()) {
return "";
}
return document.substring(off,off + len);
}
@Override
public char charAt(int index) {
if (index < 0 || index >= document.length()) {
return 0;
}
return document.charAt(index);
}
}
Pos.javapackage grid.text.index;
public class Pos {
private String target;
/** * Pos for current matched full target text */
private int pos = -1;
/** * Index in position array for current matched full target text */
int arrayIndex = -1;
private boolean found = false;
public Pos(String target) {
this.target = target;
}
public String getTarget() {
return target;
}
public int getPos() {
return pos;
}
public boolean isFound() {
return found;
}
void setPos(int pos) {
this.pos = pos;
}
void setFound(boolean found) {
this.found = found;
}
}
SimpleTextIndexer.javapackage grid.text.index;
public class SimpleTextIndexer implements TextIndexer {
private String document;
public SimpleTextIndexer(String document) {
this.document = document;
}
@Override
public int count(String text) {
int off = 0;
int count = 0;
final int len = text.length();
while ((off = document.indexOf(text,off)) > -1) {
count++;
off += len;
}
return count;
}
@Override
public Pos find(Pos pos) {
final String text = pos.getTarget();
final int len = text.length();
int off = pos.getPos() + len;
if (pos.getPos() < 0)
off = 0;
pos.setFound(false);
if ((off = document.indexOf(text,off)) > -1) {
pos.setFound(true);
pos.setPos(off);
}
return pos;
}
@Override
public int len() {
return document.length();
}
@Override
public String sub(int off,int len) {
return document.substring(off,off + len);
}
@Override
public char charAt(int index) {
if (index < 0 || index >= document.length()) {
return 0;
}
return document.charAt(index);
}
}
TextIndexer.javapackage grid.text.index;
public interface TextIndexer {
/** * @param text * @return count for specific text */
public int count(String text);
/** * @param pos * @return next position for current pos */
public Pos find(Pos pos);
/** * @return original document length */
public int len();
/** * @param off * @param len * @return the sub string start from <b>off</b> and with a length with * <b>len</b> */
public String sub(int off,int len);
/** * @param index * @return return the character in the specified index */
public char charAt(int index);
}
participle(编辑:应用网_阳江站长网) 【声明】本站内容均来自网络,其相关言论仅代表作者个人观点,不代表本站立场。若无意侵犯到您的权利,请及时与联系站长删除相关内容! |


