1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
package mobvista.dmp.common;
import com.google.common.collect.Lists;
import org.apache.hadoop.io.ArrayFile;
import org.apache.hadoop.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.Iterator;
import java.util.List;
/**
* author: houying
* date : 16-10-31
* desc :
*/
public abstract class ListCache<T> implements Iterable<T> {
private static final Logger logger = LoggerFactory.getLogger(ListCache.class);
private List<T> list;
private boolean isWriteFile;
private Iterator<T> iterator;
private BufferedWriter writer;
private File file;
private boolean isClosed;
private ListCache() {
isWriteFile = false;
list = Lists.newLinkedList();
writer = null;
isClosed = false;
}
public void add(T t) throws IOException {
if (isWriteFile) {
addIntoFile(t);
} else {
addIntoList(t);
}
}
protected void addIntoList(T t) throws IOException {
if (list.size() < 1000) {
list.add(t);
} else if (list.size() == 1000){ //刚刚到达阈值，将list的元素刷入文件
logger.info("list size is larger than 1000, switch to file cache!");
if (writer == null) {
file = new File("list-cache" + System.currentTimeMillis() +".txt");
writer = new BufferedWriter(new FileWriter(file));
}
for (T element: list) {
writeLine(element, writer);
writer.newLine();
}
writeLine(t, writer);
writer.newLine();
writer.flush();
isWriteFile = true;
}
}
private void addIntoFile(T t) throws IOException {
writeLine(t, writer);
writer.newLine();
}
public void flushAndClose() throws IOException {
if (isWriteFile) {
writer.close();
}
isClosed = true;
}
@Override
public Iterator<T> iterator() {
if (!isClosed) {
throw new IllegalStateException("did not call flushAndClose() for ListCache");
}
if (iterator == null) {
if (!isWriteFile) {
iterator = list.iterator();
} else {
try {
iterator = new FileIterator(file);
} catch (FileNotFoundException e) {
e.printStackTrace();
}
}
}
return iterator;
}
public class FileIterator implements Iterator<T> {
private BufferedReader reader;
private String nextLine;
public FileIterator(File file) throws FileNotFoundException {
reader = new BufferedReader(new FileReader(file));
nextLine = null;
}
@Override
public boolean hasNext() {
try {
nextLine = reader.readLine();
} catch (IOException e) {
e.printStackTrace();
}
if (nextLine == null) {
try {
reader.close();
file.delete();
} catch (IOException e) {
e.printStackTrace();
}
return false;
} else {
return true;
}
}
@Override
public T next() {
return readLine(nextLine);
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
}
protected abstract void writeLine(T element, Writer writer) throws IOException;
protected abstract T readLine(String nextLine);
public static ListCache<String> newStringListCache() {
return new ListCache<String>() {
@Override
protected void writeLine(String element, Writer writer) throws IOException {
writer.write(element);
}
@Override
protected String readLine(String nextLine) {
return nextLine;
}
};
}
public static void main(String[] args) throws IOException {
ListCache<String> listCache = ListCache.newStringListCache();
for (int i = 0; i < 1001; i++) {
listCache.add(i + "");
}
listCache.flushAndClose();
for (String s: listCache) {
System.out.println(s);
}
}
}