SweetPotato::Plagger

■ [assets][CustomFeed-Script] Web::Scraper for CustomFeed::Script はてなハイク

新サービス・はてなハイクをスクレイプ。

新着ページ・キーワードページ・ユーザページ・Following・Profileのどれでもうまくいくはず。

まだβだからHTMLの構造が変わってスクレイパが使えなくなるかもしれんので注意。

assets/plugins/CustomFeed-Script/hatena-haiku.pl

#!/usr/local/bin/perl
# author: SweetPotato
use strict;
use warnings;

use DateTime::Format::MySQL;
use DateTime::Format::W3CDTF;
use URI;
use Web::Scraper 0.22;
use YAML;

my $url = shift or die;

my $s = scraper {
    process '//title', title => 'text';
    process '//*[@class="entry" and not(.//div[@class="input"])]',
      'entry[]' => scraper {
        process 'img.profile-image',
          icon => ['@src', sub { +{ url => $_->as_string } } ];
        process '.title', title => 'text';
        process '.body', body => 'html';
        process '.username a', author => 'text';
        process '.timestamp a', link => ['@href', sub { $_->as_string } ];
        process '.timestamp', date => ['text', \&mk_date];
        result qw( icon title body author link date );
    };
    result qw( title entry );
};
$s->user_agent->env_proxy;

my $res = $s->scrape(URI->new($url));

binmode STDOUT, ':utf8';
print YAML::Dump +{
    title => $res->{title},
    link  => $url,
    entry => $res->{entry},
};

sub mk_date {
    my $dt = DateTime::Format::MySQL->parse_datetime(shift);
       $dt->set_time_zone('Asia/Tokyo');
    DateTime::Format::W3CDTF->format_datetime($dt);
}

config.yaml

plugins:
  - module: Subscription::Config
    config:
      feed:
         - url: 'script:/path/to/hatena-haiku.pl http://h.hatena.ne.jp/'

  - module: CustomFeed::Script

  - module: Filter::ResolveRelativeLink