package com.cloudera.flume.handlers.hdfs;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.GZIPInputStream;
import org.apache.hadoop.io.compress.BZip2Codec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.log4j.Logger;
import org.junit.Test;
import com.cloudera.flume.conf.Context;
import com.cloudera.flume.conf.FlumeConfiguration;
import com.cloudera.flume.conf.SinkFactory.SinkBuilder;
import com.cloudera.flume.core.Attributes;
import com.cloudera.flume.core.Event;
import com.cloudera.flume.core.EventImpl;
import com.cloudera.flume.core.EventSink;
import com.cloudera.flume.handlers.avro.AvroJsonOutputFormat;
import com.cloudera.flume.handlers.text.SyslogEntryFormat;
import com.cloudera.flume.handlers.text.output.Log4jOutputFormat;
import com.cloudera.util.FileUtil;
final public static Logger LOG = Logger
.getLogger(TestEscapedCustomOutputDfs.class);
@Test
FlumeConfiguration conf = FlumeConfiguration.get();
conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "avrojson");
File f = FileUtil.mktempdir();
SinkBuilder builder = EscapedCustomDfsSink.builder();
EventSink snk = builder.build(new Context(), "file:///" + f.getPath()
+ "/sub-%{service}");
Event e = new EventImpl("this is a test message".getBytes());
Attributes.setString(e, "service", "foo");
snk.open();
snk.append(e);
snk.close();
ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
AvroJsonOutputFormat ajof = new AvroJsonOutputFormat();
ajof.format(exWriter, e);
exWriter.close();
String expected = new String(exWriter.toByteArray());
File fo = new File(f.getPath() + "/sub-foo");
FileReader fr = new FileReader(fo);
BufferedReader br = new BufferedReader(fr);
String read = br.readLine() + "\n";
assertEquals(expected, read);
}
@Test
FlumeConfiguration conf = FlumeConfiguration.get();
conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
File f = FileUtil.mktempdir();
SinkBuilder builder = EscapedCustomDfsSink.builder();
EventSink snk = builder.build(new Context(), "file:///" + f.getPath()
+ "/sub-%{service}");
Event e = new EventImpl("this is a test message".getBytes());
Attributes.setString(e, "service", "foo");
snk.open();
snk.append(e);
snk.close();
ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
SyslogEntryFormat fmt = new SyslogEntryFormat();
fmt.format(exWriter, e);
exWriter.close();
String expected = new String(exWriter.toByteArray());
File fo = new File(f.getPath() + "/sub-foo");
FileReader fr = new FileReader(fo);
BufferedReader br = new BufferedReader(fr);
String read = br.readLine() + "\n";
assertEquals(expected, read);
}
@Test
FlumeConfiguration conf = FlumeConfiguration.get();
conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "log4j");
File f = FileUtil.mktempdir();
SinkBuilder builder = EscapedCustomDfsSink.builder();
EventSink snk = builder.build(new Context(), "file:///" + f.getPath()
+ "/sub-%{service}");
Event e = new EventImpl("this is a test message".getBytes());
Attributes.setString(e, "service", "foo");
snk.open();
snk.append(e);
snk.close();
ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
Log4jOutputFormat fmt = new Log4jOutputFormat();
fmt.format(exWriter, e);
exWriter.close();
String expected = new String(exWriter.toByteArray());
File fo = new File(f.getPath() + "/sub-foo");
FileReader fr = new FileReader(fo);
BufferedReader br = new BufferedReader(fr);
String read = br.readLine() + "\n";
assertEquals(expected, read);
}
@Test
FlumeConfiguration conf = FlumeConfiguration.get();
conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_CODEC, "GzipCodec");
File f = FileUtil.mktempdir();
SinkBuilder builder = EscapedCustomDfsSink.builder();
EventSink snk = builder.build(new Context(), "file:///" + f.getPath()
+ "/sub-%{service}");
Event e = new EventImpl("this is a test message".getBytes());
Attributes.setString(e, "service", "foo");
snk.open();
snk.append(e);
snk.close();
ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
SyslogEntryFormat fmt = new SyslogEntryFormat();
fmt.format(exWriter, e);
exWriter.close();
String expected = new String(exWriter.toByteArray());
GZIPInputStream gzin = new GZIPInputStream(new FileInputStream(f.getPath()
+ "/sub-foo.gz"));
byte[] buf = new byte[1];
StringBuilder output = new StringBuilder();
while ((gzin.read(buf)) > 0) {
output.append(new String(buf));
}
gzin.close();
assertEquals(expected, output.toString());
assertTrue("temp folder successfully deleted", FileUtil.rmr(f));
}
@Test
FlumeConfiguration conf = FlumeConfiguration.get();
conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_CODEC, "BZip2Codec");
File f = FileUtil.mktempdir();
SinkBuilder builder = EscapedCustomDfsSink.builder();
EventSink snk = builder.build(new Context(), "file:///" + f.getPath()
+ "/sub-%{service}");
Event e = new EventImpl("this is a test message".getBytes());
Attributes.setString(e, "service", "foo");
snk.open();
snk.append(e);
snk.close();
ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
SyslogEntryFormat fmt = new SyslogEntryFormat();
fmt.format(exWriter, e);
exWriter.close();
String expected = new String(exWriter.toByteArray());
BZip2Codec bz2Codec = new BZip2Codec();
InputStream bz2in = bz2Codec.createInputStream(new FileInputStream(f
.getPath() + "/sub-foo.bz2"));
byte[] buf = new byte[1];
StringBuilder output = new StringBuilder();
while ((bz2in.read(buf)) > 0) {
output.append(new String(buf));
}
bz2in.close();
assertEquals(expected, output.toString());
assertTrue("temp folder successfully deleted", FileUtil.rmr(f));
}
@Test
FlumeConfiguration conf = FlumeConfiguration.get();
conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_GZIP, "true");
File f = FileUtil.mktempdir();
SinkBuilder builder = EscapedCustomDfsSink.builder();
EventSink snk = builder.build(new Context(), "file:///" + f.getPath()
+ "/sub-%{service}");
Event e = new EventImpl("this is a test message".getBytes());
Attributes.setString(e, "service", "foo");
snk.open();
snk.append(e);
snk.close();
ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
SyslogEntryFormat fmt = new SyslogEntryFormat();
fmt.format(exWriter, e);
exWriter.close();
String expected = new String(exWriter.toByteArray());
GZIPInputStream gzin = new GZIPInputStream(new FileInputStream(f.getPath()
+ "/sub-foo.gz"));
byte[] buf = new byte[1];
StringBuilder output = new StringBuilder();
while ((gzin.read(buf)) > 0) {
output.append(new String(buf));
}
gzin.close();
assertEquals(expected, output.toString());
assertTrue("temp folder successfully deleted", FileUtil.rmr(f));
}
@Test(expected = IOException.class)
FlumeConfiguration conf = FlumeConfiguration.get();
conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_CODEC, "DefaultCodec");
File f = FileUtil.mktempdir();
SinkBuilder builder = EscapedCustomDfsSink.builder();
EventSink snk = builder.build(new Context(), "file:///" + f.getPath()
+ "/sub-%{service}");
Event e = new EventImpl("this is a test message".getBytes());
Attributes.setString(e, "service", "foo");
snk.open();
snk.append(e);
snk.close();
ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
SyslogEntryFormat fmt = new SyslogEntryFormat();
fmt.format(exWriter, e);
exWriter.close();
String expected = new String(exWriter.toByteArray());
DefaultCodec defaultCodec = new DefaultCodec();
InputStream defaultIn = defaultCodec.createInputStream(new FileInputStream(
f.getPath() + "/sub-foo.deflate"));
byte[] buf = new byte[1];
StringBuilder output = new StringBuilder();
while ((defaultIn.read(buf)) > 0) {
output.append(new String(buf));
}
assertEquals(expected, output.toString());
assertTrue("temp folder successfully deleted", FileUtil.rmr(f));
}
LOG.info(CompressionCodecFactory.getCodecClasses(FlumeConfiguration.get()));
}
}