package mobvista.dmp.common; import com.google.common.collect.Lists; import org.apache.hadoop.io.ArrayFile; import org.apache.hadoop.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; import java.util.Iterator; import java.util.List; /** * author: houying * date : 16-10-31 * desc : */ public abstract class ListCache<T> implements Iterable<T> { private static final Logger logger = LoggerFactory.getLogger(ListCache.class); private List<T> list; private boolean isWriteFile; private Iterator<T> iterator; private BufferedWriter writer; private File file; private boolean isClosed; private ListCache() { isWriteFile = false; list = Lists.newLinkedList(); writer = null; isClosed = false; } public void add(T t) throws IOException { if (isWriteFile) { addIntoFile(t); } else { addIntoList(t); } } protected void addIntoList(T t) throws IOException { if (list.size() < 1000) { list.add(t); } else if (list.size() == 1000){ //刚刚到达阈值,将list的元素刷入文件 logger.info("list size is larger than 1000, switch to file cache!"); if (writer == null) { file = new File("list-cache" + System.currentTimeMillis() +".txt"); writer = new BufferedWriter(new FileWriter(file)); } for (T element: list) { writeLine(element, writer); writer.newLine(); } writeLine(t, writer); writer.newLine(); writer.flush(); isWriteFile = true; } } private void addIntoFile(T t) throws IOException { writeLine(t, writer); writer.newLine(); } public void flushAndClose() throws IOException { if (isWriteFile) { writer.close(); } isClosed = true; } @Override public Iterator<T> iterator() { if (!isClosed) { throw new IllegalStateException("did not call flushAndClose() for ListCache"); } if (iterator == null) { if (!isWriteFile) { iterator = list.iterator(); } else { try { iterator = new FileIterator(file); } catch (FileNotFoundException e) { e.printStackTrace(); } } } return iterator; } public class FileIterator implements Iterator<T> { private BufferedReader reader; private String nextLine; public FileIterator(File file) throws FileNotFoundException { reader = new BufferedReader(new FileReader(file)); nextLine = null; } @Override public boolean hasNext() { try { nextLine = reader.readLine(); } catch (IOException e) { e.printStackTrace(); } if (nextLine == null) { try { reader.close(); file.delete(); } catch (IOException e) { e.printStackTrace(); } return false; } else { return true; } } @Override public T next() { return readLine(nextLine); } @Override public void remove() { throw new UnsupportedOperationException(); } } protected abstract void writeLine(T element, Writer writer) throws IOException; protected abstract T readLine(String nextLine); public static ListCache<String> newStringListCache() { return new ListCache<String>() { @Override protected void writeLine(String element, Writer writer) throws IOException { writer.write(element); } @Override protected String readLine(String nextLine) { return nextLine; } }; } public static void main(String[] args) throws IOException { ListCache<String> listCache = ListCache.newStringListCache(); for (int i = 0; i < 1001; i++) { listCache.add(i + ""); } listCache.flushAndClose(); for (String s: listCache) { System.out.println(s); } } }