|
||||||||||
| PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
| SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD | |||||||||
java.lang.Objectorg.archive.crawler.io.CrawlerJournal
public class CrawlerJournal
Utility class for a crawler journal/log that is compressed and rotates by serial number at checkpoints.
| Field Summary | |
|---|---|
protected it.unimi.dsi.mg4j.util.MutableString |
accumulatingBuffer
Allocate a buffer for accumulating lines to write and reuse it. |
static java.lang.String |
GZIP_SUFFIX
suffix to recognize gzipped files |
protected java.io.File |
gzipFile
File we're writing journal to. |
protected long |
lines
line count |
static java.lang.String |
LOG_ERROR
prefix for error lines |
static java.lang.String |
LOG_TIMESTAMP
prefix for timestamp lines |
protected java.io.Writer |
out
Stream on which we record frontier events. |
protected int |
timestamp_interval
number of lines between timestamps |
| Constructor Summary | |
|---|---|
CrawlerJournal(java.io.File file)
Create a new crawler journal at the given location |
|
CrawlerJournal(java.lang.String path,
java.lang.String filename)
Create a new crawler journal at the given location |
|
| Method Summary | |
|---|---|
void |
checkpoint(java.io.File checkpointDir)
Handle a checkpoint by rotating the current log to a checkpoint-named file and starting a new log. |
void |
close()
Flush and close the underlying IO objects. |
protected void |
considerTimestamp()
Write a timestamp line if appropriate |
static java.io.BufferedInputStream |
getBufferedInput(java.io.File source)
Get a BufferedInputStream on the recovery file given. |
static java.io.BufferedReader |
getBufferedReader(java.io.File source)
Get a BufferedReader on the crawler journal given |
static java.io.BufferedReader |
getBufferedReader(java.net.URL source)
Get a BufferedReader on the crawler journal given. |
protected java.io.Writer |
initialize(java.io.File f)
|
protected void |
noteLine()
Count and note a line |
void |
seriousError(java.lang.String err)
Note a serious error vioa a special log line |
void |
writeLine(it.unimi.dsi.mg4j.util.MutableString mstring)
Write a line. |
void |
writeLine(java.lang.String string)
Write a line |
void |
writeLine(java.lang.String s1,
java.lang.String s2)
Write a line of two strings |
void |
writeLine(java.lang.String s1,
java.lang.String s2,
java.lang.String s3)
Write a line of three strings |
| Methods inherited from class java.lang.Object |
|---|
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
| Field Detail |
|---|
public static final java.lang.String LOG_ERROR
public static final java.lang.String LOG_TIMESTAMP
protected java.io.Writer out
protected long lines
protected int timestamp_interval
public static final java.lang.String GZIP_SUFFIX
protected java.io.File gzipFile
protected it.unimi.dsi.mg4j.util.MutableString accumulatingBuffer
| Constructor Detail |
|---|
public CrawlerJournal(java.lang.String path,
java.lang.String filename)
throws java.io.IOException
path - Directory to make thejournal in.filename - Name to use for journal file.
java.io.IOException
public CrawlerJournal(java.io.File file)
throws java.io.IOException
file - path at which to make journal
java.io.IOException| Method Detail |
|---|
public static java.io.BufferedReader getBufferedReader(java.io.File source)
throws java.io.IOException
source - File journal
java.io.IOException
public static java.io.BufferedReader getBufferedReader(java.net.URL source)
throws java.io.IOException
source - URL journal
java.io.IOException
public static java.io.BufferedInputStream getBufferedInput(java.io.File source)
throws java.io.IOException
source - file to open
java.io.IOException
protected java.io.Writer initialize(java.io.File f)
throws java.io.FileNotFoundException,
java.io.IOException
java.io.FileNotFoundException
java.io.IOExceptionpublic void writeLine(java.lang.String string)
string - String
public void writeLine(java.lang.String s1,
java.lang.String s2)
s1 - Strings2 - String
public void writeLine(java.lang.String s1,
java.lang.String s2,
java.lang.String s3)
s1 - Strings2 - Strings3 - Stringpublic void writeLine(it.unimi.dsi.mg4j.util.MutableString mstring)
mstring - MutableString to write
protected void noteLine()
throws java.io.IOException
java.io.IOException
protected void considerTimestamp()
throws java.io.IOException
java.io.IOExceptionpublic void close()
public void seriousError(java.lang.String err)
err -
public void checkpoint(java.io.File checkpointDir)
throws java.io.IOException
checkpointDir -
java.io.IOException
|
||||||||||
| PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
| SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD | |||||||||