Skip to content

etcart / RIVet

Go to a project
Commit 47297b52 by etcart
Options

improved lex pointer system

parent ad4b27c9
Inline Side-by-side
Showing with 319 additions and 142 deletions
RIVLower.h
...@@ -47,6 +47,7 @@ ...@@ -47,6 +47,7 @@
/* the size of the tempBlock used in consolidation and implicit RIVs */ /* the size of the tempBlock used in consolidation and implicit RIVs */
#define TEMPSIZE 3*RIVSIZE #define TEMPSIZE 3*RIVSIZE
/* the sparseRIV is a RIV form optimized for RIVs that will be mostly 0s /* the sparseRIV is a RIV form optimized for RIVs that will be mostly 0s
* as this is often an ideal case, it is adviseable as the default * as this is often an ideal case, it is adviseable as the default
* unless we are doing long term RIV aggregation. * unless we are doing long term RIV aggregation.
...@@ -54,6 +55,7 @@ ...@@ -54,6 +55,7 @@
* containing locations and values, where pairs are found in like array * containing locations and values, where pairs are found in like array
* indices. * indices.
*/ */
typedef struct{ typedef struct{
char name[100]; char name[100];
int *values; int *values;
...@@ -70,7 +72,7 @@ typedef struct{ ...@@ -70,7 +72,7 @@ typedef struct{
*/ */
typedef struct{ typedef struct{
char name[100]; char name[100];
int cached; void* cached;
int frequency; int frequency;
int contextSize; int contextSize;
float magnitude; float magnitude;
...@@ -85,7 +87,8 @@ struct RIVData{ ...@@ -85,7 +87,8 @@ struct RIVData{
int h_tempBlock[TEMPSIZE]; int h_tempBlock[TEMPSIZE];
int tempSize; int tempSize;
char lexName[255]; char lexName[255];
denseRIV* RIVCache[CACHESIZE]; denseRIV** RIVCache;
char flags;
}static RIVKey; }static RIVKey;
/*consolidateD2S takes a denseRIV value-set input, and returns a sparse RIV with /*consolidateD2S takes a denseRIV value-set input, and returns a sparse RIV with
......
RIVLower.h.gch
No preview for this file type
RIVclasses
No preview for this file type
RIVclasses.c
#include <stdio.h> #include <stdio.h>
#define CACHESIZE 0
#define CACHEEXCLUSIVE 1
#define RIVSIZE 50000 #define RIVSIZE 50000
#include "RIVtools.h" #include "RIVtools.h"
char* clean(char* word);
char* stem(char* word); char* stem(char* word);
typedef char label[200];
struct RIVclass{
label name;
sparseRIV* set;
int setSize;
};
int main(){ int main(){
lexOpen("consolidatedLexicon50-8");
FILE* text = fopen("../books/pg56902.txt", "r"); lexOpen("lexicon", "rx");
if(!text){
puts("no file"); int classNo = 0;
return 1; label className = "tempName";
} label* classNames = calloc(1, sizeof(label));
denseRIV accumulate = {0}; int classCount = 0;
sparseRIV temp;
char word[100]; struct RIVclass* classes = malloc(sizeof(struct RIVclass));
while(fscanf(text, "%99s", word)){
if(feof(text)) break; strcpy(classes[classCount].name, className);
if(!*word) break; strcpy(classNames[classCount], className);
classCount++;
while(1){
if(stem(word)){ FILE* text = fopen("../bookCleaner/cleanbooks/pg56902clean.txt", "r");
if(!text){
puts("no file");
return 1;
}
denseRIV accumulate = {0};
sparseRIV temp;
char word[100];
while(fscanf(text, "%99s", word)){
if(feof(text)) break;
if(!*word) break;
if(!*clean(word)) continue;
//if(stem(word)){
denseRIV* wordRIV = lexPull(word); denseRIV* wordRIV = lexPull(word);
if(!wordRIV){ if(!wordRIV){
printf("%s, not in lexicon\n", word); //printf("%s, not in lexicon\n", word);
continue; continue;
}else{ }else{
//printf("%s, succesfully pulled\n", word);
temp = consolidateD2S(wordRIV->values); temp = consolidateD2S(wordRIV->values);
addS2D(accumulate.values, temp); addS2D(accumulate.values, temp);
free(temp.locations); free(temp.locations);
free(wordRIV); free(wordRIV);
} }
}else{ }
printf("%s, not in wordNet\n", word);
struct RIVclass* class = classes+classNo;
class->set = malloc(sizeof(sparseRIV));
class->setSize = 0;
class->set[class->setSize] = consolidateD2S(accumulate.values);
class->setSize++;
char* labelTemp = strstr(*classNames, className);
if(!labelTemp){
classNames = realloc(classNames, classCount*sizeof(label));
strcpy(classNames[classCount], className);
classCount++;
}else{
classNo = (labelTemp-*classNames);
} }
} }
return 0; return 0;
} }
char* clean(char* word){
char* letter = word;
char output[100] = {0};
char *outLetter = output;
while(*letter){
if(*letter >= 'A' && *letter <= 'Z'){
*outLetter = *letter + 32;
outLetter++;
}else if( *letter >= 'a' && *letter <= 'z'){
*outLetter = *letter;
outLetter++;
}
letter++;
}
strcpy(word,output);
return word;
}
char* stem(char* word){ char* stem(char* word){
...@@ -61,10 +115,15 @@ char* stem(char* word){ ...@@ -61,10 +115,15 @@ char* stem(char* word){
fscanf(WNfile, "%d", &WNdata); fscanf(WNfile, "%d", &WNdata);
if(!WNdata) return NULL; if(!WNdata) {
if(WNdata == 1) return word;
fclose(WNfile);
return NULL;
}
if(WNdata == 1) {
fclose(WNfile);
return word;
}
if(WNdata == 2){ if(WNdata == 2){
fscanf(WNfile, "%s", word); fscanf(WNfile, "%s", word);
fclose(WNfile); fclose(WNfile);
......
RIVclasses.o
No preview for this file type
RIVlexicon.h
...@@ -5,9 +5,39 @@ ...@@ -5,9 +5,39 @@
#include "RIVaccessories.h" #include "RIVaccessories.h"
#include "assert.h" #include "assert.h"
#ifndef CACHEEXCLUSIVE #ifndef READFLAG
#define CACHEEXCLUSIVE 0 #define READFLAG 0x01
#endif #endif
#ifndef WRITEFLAG
#define WRITEFLAG 0x02
#endif
#ifndef INCFLAG
#define INCFLAG 0x04
#endif
#ifndef CACHEFLAG
#define CACHEFLAG 0x08
#endif
#ifndef SORTCACHE
#ifndef HASHCACHE
#define HASHCACHE
#endif
#endif
typedef struct{
char lexName[100];
denseRIV* *cache;
struct cacheList* listPoint;
char flags;
}LEXICON;
struct cacheList{
denseRIV* *cache;
struct cacheList* next;
struct cacheList* prev;
}*rootCache = NULL;
#define IODISPLACEMENT (sizeof(((sparseRIV*)0)->count)\ #define IODISPLACEMENT (sizeof(((sparseRIV*)0)->count)\
+ sizeof(((sparseRIV*)0)->frequency)\ + sizeof(((sparseRIV*)0)->frequency)\
+ sizeof(((sparseRIV*)0)->contextSize)\ + sizeof(((sparseRIV*)0)->contextSize)\
...@@ -15,17 +45,18 @@ ...@@ -15,17 +45,18 @@
/ sizeof(int) / sizeof(int)
int* IOstagingSlot = RIVKey.h_tempBlock+RIVSIZE; //#TODO format this better int* IOstagingSlot = RIVKey.h_tempBlock+RIVSIZE; //#TODO format this better
/* lexOpen is called to "open the lexicon", setting up for later calls to /* lexOpen is called to "open the lexicon", setting up for later calls to
* lexPush and lexPull. if the lexicon has not been opened before calls * lexPush and lexPull. if the lexicon has not been opened before calls
* to these functions, their behavior can be unpredictable, most likely crashing * to these functions, their behavior can be unpredictable, most likely crashing
*/ */
void lexOpen(); LEXICON* lexOpen(const char* lexName, const char* flags);
/* lexClose should always be called after the last lex push or lex pull call /* lexClose should always be called after the last lex push or lex pull call
* if the lexicon is left open, some vector data may be lost due to * if the lexicon is left open, some vector data may be lost due to
* un-flushed RIV cache * un-flushed RIV cache
*/ */
void lexClose(); void lexClose(LEXICON*);
/* both lexPush and lexPull must be called *after* the lexOpen() function /* both lexPush and lexPull must be called *after* the lexOpen() function
...@@ -33,34 +64,34 @@ void lexClose(); ...@@ -33,34 +64,34 @@ void lexClose();
* data security */ * data security */
/* lexPush writes a denseRIV to the lexicon for permanent storage */ /* lexPush writes a denseRIV to the lexicon for permanent storage */
int lexPush(denseRIV* RIVout); int lexPush(LEXICON* lexicon, denseRIV* RIVout);
/* cacheCheckOnPush tests the state of this vector in our lexicon cache /* cacheCheckOnPush tests the state of this vector in our lexicon cache
* and returns 1 on "success" indicating cache storage and no need to push to file * and returns 1 on "success" indicating cache storage and no need to push to file
* or returns 0 on "failure" indicating that the vector need be pushed to file * or returns 0 on "failure" indicating that the vector need be pushed to file
*/ */
int cacheCheckOnPush(denseRIV* RIVout); int cacheCheckOnPush(LEXICON* lexicon, denseRIV* RIVout);
/* lexPull reads a denseRIV from the lexicon, under "word" /* lexPull reads a denseRIV from the lexicon, under "word"
* if the file does not exist, it creates a 0 vector with the name of word * if the file does not exist, it creates a 0 vector with the name of word
* lexPull returns a denseRIV *pointer* because its data must be tracked * lexPull returns a denseRIV *pointer* because its data must be tracked
* globally for key optimizations * globally for key optimizations
*/ */
denseRIV* lexPull(char* word); denseRIV* lexPull(LEXICON* lexicon, char* word);
/* cacheCheckonPull checks if the word's vector is stored in cache, /* cacheCheckonPull checks if the word's vector is stored in cache,
* and returns a pointer to that vector on success * and returns a pointer to that vector on success
* or returns a NULL pointer if the word is not cached, indicating a need * or returns a NULL pointer if the word is not cached, indicating a need
* to pull from file * to pull from file
*/ */
denseRIV* cacheCheckOnPull(char* word); denseRIV* cacheCheckOnPull(LEXICON* lexicon, char* word);
/* fLexPush pushes the data contained in a denseRIV out to a lexicon file, /* fLexPush pushes the data contained in a denseRIV out to a lexicon file,
* saving it for long-term aggregation. function is called by "lexPush", * saving it for long-term aggregation. function is called by "lexPush",
* which is what users should actually use. lexPush, unlike fLexPush, * which is what users should actually use. lexPush, unlike fLexPush,
* has cache logic under the hood for speed and harddrive optimization * has cache logic under the hood for speed and harddrive optimization
*/ */
int fLexPush(denseRIV* RIVout); int fLexPush(LEXICON* lexicon, denseRIV* RIVout);
/* flexPull pulls data directly from a file and converts it (if necessary) /* flexPull pulls data directly from a file and converts it (if necessary)
* to a denseRIV. function is called by "lexPull" which is what users * to a denseRIV. function is called by "lexPull" which is what users
...@@ -77,63 +108,187 @@ void signalSecure(int signum, siginfo_t *si, void* arg); ...@@ -77,63 +108,187 @@ void signalSecure(int signum, siginfo_t *si, void* arg);
*/ */
int saturationForStaging(denseRIV* output); int saturationForStaging(denseRIV* output);
/* begin definitions */ /* begin definitions */
void lexOpen(char* lexName){ LEXICON* lexOpen(const char* lexName, const char* flags){
LEXICON* output = calloc(1, sizeof(LEXICON));
/* identify the presence of read, write, and exclusive flags */
char* r = strstr(flags, "r");
char* w = strstr(flags, "w");
char* x = strstr(flags, "x");
struct stat st = {0}; struct stat st = {0};
if (stat(lexName, &st) == -1) {
mkdir(lexName, 0777);
} if(w){
strcpy(RIVKey.lexName, lexName); /* if set to write, we check and create if necessary, the lexicon */
/* open a slot at least large enough for ;worst case handling of if (stat(lexName, &st) == -1) {
* sparse to dense conversion. may be enlarged by filetoL2 functions */ mkdir(lexName, 0777);
struct sigaction action = {0}; }
action.sa_sigaction = signalSecure; output->flags |= WRITEFLAG;
action.sa_flags = SA_SIGINFO; }else if(r){
for(int i=1; i<27; i++){ /* if set to read and not write, return null if lexicon does not exist */
sigaction(i,&action,NULL); if (stat(lexName, &st) == -1) {
free(output);
return NULL;
}
output->flags |= READFLAG;
}
/* if not set to exclusive, set the inclusive flag */
if(!x){
output->flags |= INCFLAG;
} }
strcpy(output->lexName, lexName);
#if CACHESIZE > 0
if(r && w){
//#TODO include hash vs sort cache logic flags
/* if we will be reading and writing the same lexicon, setup a
* cache for this lexicon to speed up rewrites */
struct cacheList* newCache = calloc(1, sizeof(struct cacheList));
newCache->cache = calloc(CACHESIZE, sizeof(denseRIV*));
output->flags |= CACHEFLAG;
output->cache = newCache->cache;
newCache->next = rootCache;
if(rootCache){
rootCache->prev = newCache;
}
rootCache = newCache;
output->listPoint = newCache;
struct sigaction action = {0};
action.sa_sigaction = signalSecure;
action.sa_flags = SA_SIGINFO;
for(int i=1; i<27; i++){
sigaction(i,&action,NULL);
}
}
#endif
/* open a slot for a cache of dense RIVs, optimized for frequent accesses */ return output;
memset(RIVKey.RIVCache, 0, sizeof(denseRIV*)*CACHESIZE); }
void lexClose(LEXICON* toClose){
#if CACHESIZE>0
if(toClose->flags & CACHEFLAG){
if(cacheDump(toClose->cache)){
puts("cache dump failed, some lexicon data was lost");
}
struct cacheList* listPoint = toClose->listPoint;
if(listPoint->prev){
listPoint->prev->next = toClose->listPoint->next;
}
if(listPoint->next){
listPoint->next->prev = toClose->listPoint->prev;
}
free(listPoint);
}
#endif
free(toClose);
} }
void lexClose(){
int cacheDump(denseRIV* *toDump){
int flag = 0;
if(cacheDump()){ denseRIV* *toDump_slider = toDump;
puts("cache dump failed, some lexicon data was lost"); denseRIV* *toDump_stop = toDump+CACHESIZE;
while(toDump_slider<toDump_stop){
if(*toDump_slider){
flag += fLexPush((LEXICON*)(*toDump_slider)->cached,*toDump_slider);
}
toDump_slider++;
} }
free(toDump);
return flag;
} }
#if CACHESIZE > 0 #if CACHESIZE > 0
denseRIV* cacheCheckOnPull(char* word){ denseRIV* cacheCheckOnPull(LEXICON* lexicon, char* word){
#ifdef HASHCACHE
srand(wordtoSeed(word)); srand(wordtoSeed(word));
int hash = rand()%CACHESIZE; int hash = rand()%CACHESIZE;
if(RIVKey.RIVCache[hash]){ if(lexicon->cache[hash]){
if(!strcmp(word, RIVKey.RIVCache[hash]->name)){ if(!strcmp(word, lexicon->cache[hash]->name)){
/* if word is cached, pull from cache and exit */ /* if word is cached, pull from cache and exit */
return RIVKey.RIVCache[hash]; return lexicon->cache[hash];
} }
} }
return NULL; return NULL;
#endif
#ifdef SORTCACHE
#endif
}
#if CACHESIZE > 0
int cacheCheckOnPush(LEXICON* lexicon, denseRIV* RIVout){
/* if our RIV was cached already, no need to play with it */
if(RIVout->cached == lexicon){
return 1;
}
#if HASHCACHE
srand(wordtoSeed(RIVout->name));
int hash = rand()%CACHESIZE;
/* if there is no word in this cache slot */
if(!lexicon->cache[hash]){
/* push to cache instead of file */
lexicon->cache[hash] = RIVout;
lexicon->cache[hash]->cached = lexicon;
return 1;
/*if the current RIV is more frequent than the RIV holding its slot */
}
if(RIVout->frequency > lexicon->cache[hash]->frequency ){
/* push the lower frequency cache entry to a file */
fLexPush(lexicon, lexicon->cache[hash]);
/* replace this cache-slot with the current vector */
lexicon->cache[hash] = RIVout;
lexicon->cache[hash]->cached = lexicon;
return 1;
}
return 0;
#endif
#if SORTCACHE
#endif
} }
#endif #endif
denseRIV* lexPull(char* word){ #endif
denseRIV* lexPull(LEXICON* lexicon, char* word){
denseRIV* output = NULL; denseRIV* output = NULL;
#if CACHESIZE > 0 #if CACHESIZE > 0
if(lexicon->flags & CACHEFLAG){
/* if there is a cache, first check if the word is cached */ /* if there is a cache, first check if the word is cached */
if((output = cacheCheckOnPull(word))){ if((output = cacheCheckOnPull(lexicon, word))){
return output; return output;
}
} }
#endif /* CACHESIZE > 0 */ #endif /* CACHESIZE > 0 */
/* if not, attempt to pull the word data from lexicon file */ /* if not, attempt to pull the word data from lexicon file */
char pathString[200]; char pathString[200];
sprintf(pathString, "%s/%s", RIVKey.lexName, word); sprintf(pathString, "%s/%s", lexicon->lexName, word);
FILE *lexWord = fopen(pathString, "rb"); FILE *lexWord = fopen(pathString, "rb");
...@@ -145,11 +300,12 @@ denseRIV* lexPull(char* word){ ...@@ -145,11 +300,12 @@ denseRIV* lexPull(char* word){
strcpy(output->name, word); strcpy(output->name, word);
fclose(lexWord); fclose(lexWord);
}else{ }else{
#if CACHEEXCLUSIVE == 0 /* if lexicon is set to inclusive (can gain new words) */
/*if file does not exist, return a 0 vector (word is new to the lexicon */ if(lexicon->flags & INCFLAG){
output = calloc(1, sizeof(denseRIV)); /*if file does not exist, return a 0 vector (word is new to the lexicon) */
strcpy(output->name, word); output = calloc(1, sizeof(denseRIV));
#endif strcpy(output->name, word);
}
/*if lexicon is set to exclusive, will return a NULL pointer instead of a 0 vector */ /*if lexicon is set to exclusive, will return a NULL pointer instead of a 0 vector */
} }
...@@ -157,53 +313,23 @@ denseRIV* lexPull(char* word){ ...@@ -157,53 +313,23 @@ denseRIV* lexPull(char* word){
return output; return output;
} }
#if CACHESIZE > 0
int cacheCheckOnPush(denseRIV* RIVout){
/* if our RIV was cached already, no need to play with it */
if(RIVout->cached){
return 1;
}
srand(wordtoSeed(RIVout->name));
int hash = rand()%CACHESIZE;
/* if there is no word in this cache slot */
if(!RIVKey.RIVCache[hash]){
/* push to cache instead of file */
RIVKey.RIVCache[hash] = RIVout;
RIVKey.RIVCache[hash]->cached = 1;
return 1;
/*if the current RIV is more frequent than the RIV holding its slot */
}
if(RIVout->frequency > RIVKey.RIVCache[hash]->frequency ){
/* push the lower frequency cache entry to a file */
fLexPush(RIVKey.RIVCache[hash]);
/* replace this cache-slot with the current vector */
RIVKey.RIVCache[hash] = RIVout; int lexPush(LEXICON* lexicon, denseRIV* RIVout){
RIVKey.RIVCache[hash]->cached = 1;
return 1;
}
return 0;
}
#endif
int lexPush(denseRIV* RIVout){
#if CACHESIZE > 0 #if CACHESIZE > 0
if(lexicon->flags & CACHEFLAG){
/* check the cache to see if it belongs in cache */ /* check the cache to see if it belongs in cache */
if(cacheCheckOnPush(RIVout)){ if(cacheCheckOnPush(lexicon, RIVout)){
/* if the cache check returns 1, it has been dealth with in cache */ /* if the cache check returns 1, it has been dealth with in cache */
return 0; return 0;
}
} }
#endif /* CACHESIZE != 0 */ #endif
/* push to the lexicon */ /* push to the lexicon */
return fLexPush(RIVout); return fLexPush(lexicon, RIVout);
} }
...@@ -242,12 +368,12 @@ int saturationForStaging(denseRIV* output){ ...@@ -242,12 +368,12 @@ int saturationForStaging(denseRIV* output){
return *count; return *count;
} }
int fLexPush(denseRIV* output){ int fLexPush(LEXICON* lexicon, denseRIV* output){
char pathString[200] = {0}; char pathString[200] = {0};
/* word data will be placed in a (new?) file under the lexicon directory /* word data will be placed in a (new?) file under the lexicon directory
* in a file named after the word itself */ * in a file named after the word itself */
sprintf(pathString, "%s/%s", RIVKey.lexName, output->name); sprintf(pathString, "%s/%s", lexicon->lexName, output->name);
int saturation = saturationForStaging(output); int saturation = saturationForStaging(output);
...@@ -267,7 +393,7 @@ int fLexPush(denseRIV* output){ ...@@ -267,7 +393,7 @@ int fLexPush(denseRIV* output){
printf("lexicon push has failed for word: %s\nconsider cleaning inputs", output->name); printf("lexicon push has failed for word: %s\nconsider cleaning inputs", output->name);
return 1; return 1;
} }
fwrite(&output->cached, sizeof(int), RIVSIZE+4, lexWord); fwrite(((int*)&output->cached)+1, sizeof(int), RIVSIZE+4, lexWord);
fclose(lexWord); fclose(lexWord);
} }
...@@ -364,28 +490,16 @@ denseRIV* fLexPull(FILE* lexWord){ ...@@ -364,28 +490,16 @@ denseRIV* fLexPull(FILE* lexWord){
} }
void signalSecure(int signum, siginfo_t *si, void* arg){
while(rootCache){
int cacheDump(){ if(cacheDump(rootCache->cache)){
puts("cache dump failed, some lexicon data lost");
int flag = 0;
for(int i = 0; i < CACHESIZE; i++){
if(RIVKey.RIVCache[i]){
flag += fLexPush(RIVKey.RIVCache[i]);
} }
rootCache = rootCache->next;
free(rootCache->prev);
} }
return flag; signal(signum, SIG_DFL);
} kill(getpid(), signum);
/*TODO add a simplified free function*/
void signalSecure(int signum, siginfo_t *si, void* arg){
if(cacheDump()){
puts("cache dump failed, some lexicon data lost");
}
signal(signum, SIG_DFL);
kill(getpid(), signum);
} }
#endif #endif
RIVlexicon.h.gch
No preview for this file type
RIVread
No preview for this file type
RIVread.c
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
#include <dirent.h> #include <dirent.h>
#include <error.h> #include <error.h>
#include <string.h> #include <string.h>
#define CACHESIZE 100000 #define CACHESIZE 10000
#include "RIVtools.h" #include "RIVtools.h"
//this program reads a directory full of files, and adds all context vectors (considering file as context) //this program reads a directory full of files, and adds all context vectors (considering file as context)
...@@ -16,13 +16,13 @@ void fileGrind(FILE* textFile); ...@@ -16,13 +16,13 @@ void fileGrind(FILE* textFile);
void addContext(denseRIV* lexRIV, sparseRIV context); void addContext(denseRIV* lexRIV, sparseRIV context);
void directoryGrind(char *rootString); void directoryGrind(char *rootString);
void lineGrind(char* textLine); void lineGrind(char* textLine);
LEXICON* lp;
int main(int argc, char *argv[]){ int main(int argc, char *argv[]){
char pathString[1000]; char pathString[1000];
lp = lexOpen("lexicon", "rw");
//we open the lexicon, if it does not yet exist, it will be created //we open the lexicon, if it does not yet exist, it will be created
lexOpen("lexicon");
//we format the root directory, preparing to scan its contents //we format the root directory, preparing to scan its contents
...@@ -39,7 +39,7 @@ int main(int argc, char *argv[]){ ...@@ -39,7 +39,7 @@ int main(int argc, char *argv[]){
directoryGrind(pathString); directoryGrind(pathString);
//we close the lexicon again, ensuring all data is secured //we close the lexicon again, ensuring all data is secured
lexClose(); lexClose(lp);
return 0; return 0;
} }
...@@ -81,6 +81,7 @@ void directoryGrind(char *rootString){ ...@@ -81,6 +81,7 @@ void directoryGrind(char *rootString){
fclose(input); fclose(input);
} }
} }
closedir(directory);
} }
...@@ -126,7 +127,7 @@ void lineGrind(char* textLine){ ...@@ -126,7 +127,7 @@ void lineGrind(char* textLine){
//we pull the vector corresponding to each word from the lexicon //we pull the vector corresponding to each word from the lexicon
//if it's a new word, lexPull returns a 0 vector //if it's a new word, lexPull returns a 0 vector
lexiconRIV= lexPull(word); lexiconRIV= lexPull(lp, word);
//we add the context of this file to this wordVector //we add the context of this file to this wordVector
addContext(lexiconRIV, contextVector); addContext(lexiconRIV, contextVector);
...@@ -138,7 +139,7 @@ void lineGrind(char* textLine){ ...@@ -138,7 +139,7 @@ void lineGrind(char* textLine){
lexiconRIV->frequency += 1; lexiconRIV->frequency += 1;
//and finally we push it back to the lexicon for permanent storage //and finally we push it back to the lexicon for permanent storage
lexPush(lexiconRIV); lexPush(lp, lexiconRIV);
} }
......
RIVread.o
No preview for this file type
RIVtools.h.gch
No preview for this file type
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment