dateext

# keep 4 weeks worth of backlogs
rotate 27

# uncomment this if you want your log files compressed
compress

error_log-20150315.gz
error_log

service httpd restart

httpd を停止中:                                            [  OK  ]
httpd を起動中: httpd: Could not reliably determine the server's fully qualified domain name, using 127.0.0.1 for ServerName
                                                           [  OK  ]

127.0.0.1      hogehost localhost.localdomain localhost

#ServerName www.example.com:80
ServerName hogehost:80

rpm --import http://packages.elasticsearch.org/GPG-KEY-elasticsearch

cat >> /etc/yum.repos.d/elasticsearch.repo <<'EOF'
[elasticsearch-1.0]
name=Elasticsearch repository for 1.0.x packages
baseurl=http://packages.elasticsearch.org/elasticsearch/1.0/centos
gpgcheck=1
gpgkey=http://packages.elasticsearch.org/GPG-KEY-elasticsearch
enabled=1
EOF

yum install elasticsearch java-1.7.0-openjdk

chkconfig elasticsearch on

service elasticsearch start

useradd kibana

passwd kibana
ユーザー kibana のパスワードを変更。
新しいパスワード: [パスワード入力]
よくないパスワード: 辞書の単語に基づいています
新しいパスワードを再入力してください: [もう一度パスワード入力]
passwd: 全ての認証トークンが正しく更新できました。

chmod +x /home/kibana

su - kibana

curl -LO https://download.elasticsearch.org/kibana/kibana/kibana-3.0.0milestone5.tar.gz

tar zxvf kibana-3.0.0milestone5.tar.gz

ln -s /home/kibana/kibana-3.0.0milestone5 ./kibana

// 以下のように設定
elasticsearch: "http://(ドメイン)/es/",

exit

htdigest -c /etc/httpd/conf/htdigest "Required authentication" (Basic認証の設定したいID)
Adding password for okochang in realm Required authentication.
New password: [パスワード入力] (Basic認証の設定したいPW)
Re-type new password: [パスワード入力] (Basic認証の設定したいPW)

httpd -t

Syntax OK

service httpd restart

http://(ドメイン)/#/dashboard/file/default.json

yum install gcc gcc-c++ libcurl-devel

/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-elasticsearch --no-ri --no-rdoc

vim /etc/td-agent/td-agent.conf

## Input
<source>
  type tail
  path /var/log/httpd/access_log
  format /^(?<date>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \w{3}) (?<processing_time>[^ ]*) (?<remote>[^ ]*) (?<user>[^ ]*) \[(?<method>.*)\] (?<status>[^ ]*) (?<size>[^ ]*) \[(?<referer>[^ ]*)\] \[(?<agent>.*)\]/
  pos_file /var/log/td-agent/tmp/apache.access.log.pos
  tag apache.access
</source>

## Output
<match apache.access>
  type copy
  <store>
    type file
    path /var/log/td-agent/apache.access
    time_slice_format %Y%m%d
    time_format %Y%m%dT%H%M%S%z
  </store>
  <store>
    type forward
    send_timeout 60s
    recover_wait 10s
    heartbeat_interval 1s
    <server>
      name (fluentdサーバ)
      host (fluentdサーバIP)
      port (Port)
    </server>
  </store>
  <store>
    type elasticsearch
    host (elasticsearchサーバIP)
    port (Port)
    type_name access_log
    logstash_format true
    logstash_prefix apache_access
    logstash_dateformat %Y%m
    flush_interval 10s
  </store>
</match>

root soft nofile 65536
root hard nofile 65536

// 以下追記
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.ip_local_port_range = 10240    65535

reboot

grep "custom" /etc/httpd/conf/httpd.conf

LogFormat "%{%Y-%m-%d %T %Z}t %D %a %u [%r] %s %b [%{Referer}i] [%{User-Agent}i]" custom
CustomLog logs/access_log custom

chmod 755 /var/log/httpd

curl -L http://toolbelt.treasuredata.com/sh/install-redhat.sh | sh

<match log.**>
  # fluentd-plugin-elasticsearch利用
  type elasticsearch

  # Kibanaで利用するためにindexの形式整備
  logstash_format true

  # index の prefix 指定
  logstash_prefix demo-log

  # 転送先の Elasticsearch
  hosts localhost:9200

  # Elasticsearchに書き込む際、ドキュメントtype指定
  type_name application-log

  # buffer設定 - メモリバッファ利用
  buffer_type memory

  # チャンクサイズ 1MB
  buffer_chunk_limit 1m

  # チャンクキュー最大サイズ 128
  buffer_queue_limit 128

  # 指定秒毎にバッファをflush - 指定秒数毎にElasticsearchへの書き込みリクエスト発
行
  flush_interval 2s

  # flush失敗時、最大リトライ回数
  retry_limit 17
</match>

// tmpディレクトリ作成
mkdir /var/log/td-agent/tmp
// 所有者修正
chown td-agent.td-agent /var/log/td-agent/tmp
// サーバ起動時設定
chkconfig td-agent on
// 起動
service td-agent start