src/lyrics_fetcher.cpp

   1 /***************************************************************************
   2  *   Copyright (C) 2008-2016 by Andrzej Rybczak                            *
   3  *   electricityispower@gmail.com                                          *
   4  *                                                                         *
   5  *   This program is free software; you can redistribute it and/or modify  *
   6  *   it under the terms of the GNU General Public License as published by  *
   7  *   the Free Software Foundation; either version 2 of the License, or     *
   8  *   (at your option) any later version.                                   *
   9  *                                                                         *
  10  *   This program is distributed in the hope that it will be useful,       *
  11  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  12  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  13  *   GNU General Public License for more details.                          *
  14  *                                                                         *
  15  *   You should have received a copy of the GNU General Public License     *
  16  *   along with this program; if not, write to the                         *
  17  *   Free Software Foundation, Inc.,                                       *
  18  *   51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.              *
  19  ***************************************************************************/
  20
  21 #include "config.h"
  22 #include "curl_handle.h"
  23
  24 #ifdef HAVE_CURL_CURL_H
  25
  26 #include <cstdlib>
  27 #include <cstring>
  28 #include <boost/algorithm/string/join.hpp>
  29 #include <boost/algorithm/string/replace.hpp>
  30 #include <boost/algorithm/string/split.hpp>
  31 #include <boost/algorithm/string/trim.hpp>
  32 #include <boost/regex.hpp>
  33
  34 #include "charset.h"
  35 #include "lyrics_fetcher.h"
  36 #include "utility/html.h"
  37 #include "utility/string.h"
  38
  39 LyricsFetcher *lyricsPlugins[] =
  40 {
  41         new LyricwikiFetcher(),
  42         new GeniusLyricsFetcher(),
  43         new AzLyricsFetcher(),
  44         new Sing365Fetcher(),
  45         new LyricsmaniaFetcher(),
  46         new MetrolyricsFetcher(),
  47         new JustSomeLyricsFetcher(),
  48         new InternetLyricsFetcher(),
  49         0
  50 };
  51
  52 const char LyricsFetcher::msgNotFound[] = "Not found";
  53
  54 LyricsFetcher::Result LyricsFetcher::fetch(const std::string &artist, const std::string &title)
  55 {
  56         Result result;
  57         result.first = false;
  58
  59         std::string url = urlTemplate();
  60         boost::replace_all(url, "%artist%", artist);
  61         boost::replace_all(url, "%title%", title);
  62
  63         std::string data;
  64         CURLcode code = Curl::perform(data, url);
  65
  66         if (code != CURLE_OK)
  67         {
  68                 result.second = curl_easy_strerror(code);
  69                 return result;
  70         }
  71
  72         auto lyrics = getContent(regex(), data);
  73
  74         if (lyrics.empty() || notLyrics(data))
  75         {
  76                 result.second = msgNotFound;
  77                 return result;
  78         }
  79
  80         data.clear();
  81         for (auto it = lyrics.begin(); it != lyrics.end(); ++it)
  82         {
  83                 postProcess(*it);
  84                 if (!it->empty())
  85                 {
  86                         data += *it;
  87                         if (it != lyrics.end()-1)
  88                                 data += "\n\n----------\n\n";
  89                 }
  90         }
  91
  92         result.second = data;
  93         result.first = true;
  94         return result;
  95 }
  96
  97 std::vector<std::string> LyricsFetcher::getContent(const char *regex_, const std::string &data)
  98 {
  99         std::vector<std::string> result;
 100         boost::regex rx(regex_);
 101         auto first = boost::sregex_iterator(data.begin(), data.end(), rx);
 102         auto last = boost::sregex_iterator();
 103         for (; first != last; ++first)
 104                 result.push_back(first->str(1));
 105         return result;
 106 }
 107
 108 void LyricsFetcher::postProcess(std::string &data) const
 109 {
 110         data = unescapeHtmlUtf8(data);
 111         stripHtmlTags(data);
 112         // Remove indentation from each line and collapse multiple newlines into one.
 113         std::vector<std::string> lines;
 114         boost::split(lines, data, boost::is_any_of("\n"));
 115         for (auto &line : lines)
 116                 boost::trim(line);
 117         std::unique(lines.begin(), lines.end(), [](std::string &a, std::string &b) {
 118                 return a.empty() && b.empty();
 119         });
 120         data = boost::algorithm::join(lines, "\n");
 121         boost::trim(data);
 122 }
 123
 124 /***********************************************************************/
 125
 126 LyricsFetcher::Result LyricwikiFetcher::fetch(const std::string &artist, const std::string &title)
 127 {
 128         LyricsFetcher::Result result = LyricsFetcher::fetch(artist, title);
 129         if (result.first == true)
 130         {
 131                 result.first = false;
 132
 133                 std::string data;
 134                 CURLcode code = Curl::perform(data, result.second, "", true);
 135
 136                 if (code != CURLE_OK)
 137                 {
 138                         result.second = curl_easy_strerror(code);
 139                         return result;
 140                 }
 141
 142                 auto lyrics = getContent("<div class='lyricbox'>(.*?)</div>", data);
 143
 144                 if (lyrics.empty())
 145                 {
 146                         result.second = msgNotFound;
 147                         return result;
 148                 }
 149                 std::transform(lyrics.begin(), lyrics.end(), lyrics.begin(), unescapeHtmlUtf8);
 150                 bool license_restriction = std::any_of(lyrics.begin(), lyrics.end(), [](const std::string &s) {
 151                         return s.find("Unfortunately, we are not licensed to display the full lyrics for this song at the moment.") != std::string::npos;
 152                 });
 153                 if (license_restriction)
 154                 {
 155                         result.second = "Licence restriction";
 156                         return result;
 157                 }
 158
 159                 data.clear();
 160                 for (auto it = lyrics.begin(); it != lyrics.end(); ++it)
 161                 {
 162                         stripHtmlTags(*it);
 163                         boost::trim(*it);
 164                         if (!it->empty())
 165                         {
 166                                 data += *it;
 167                                 if (it != lyrics.end()-1)
 168                                         data += "\n\n----------\n\n";
 169                         }
 170                 }
 171
 172                 result.second = data;
 173                 result.first = true;
 174         }
 175         return result;
 176 }
 177
 178 bool LyricwikiFetcher::notLyrics(const std::string &data) const
 179 {
 180         return data.find("action=edit") != std::string::npos;
 181 }
 182
 183 /**********************************************************************/
 184
 185 LyricsFetcher::Result GoogleLyricsFetcher::fetch(const std::string &artist, const std::string &title)
 186 {
 187         Result result;
 188         result.first = false;
 189
 190         std::string search_str = artist;
 191         search_str += "+";
 192         search_str += title;
 193         search_str += "+%2B";
 194         search_str += siteKeyword();
 195
 196         std::string google_url = "http://www.google.com/search?hl=en&ie=UTF-8&oe=UTF-8&q=";
 197         google_url += search_str;
 198         google_url += "&btnI=I%27m+Feeling+Lucky";
 199
 200         std::string data;
 201         CURLcode code = Curl::perform(data, google_url, google_url);
 202
 203         if (code != CURLE_OK)
 204         {
 205                 result.second = curl_easy_strerror(code);
 206                 return result;
 207         }
 208
 209         auto urls = getContent("<A HREF=\"(.*?)\">here</A>", data);
 210
 211         if (urls.empty() || !isURLOk(urls[0]))
 212         {
 213                 result.second = msgNotFound;
 214                 return result;
 215         }
 216
 217         data = unescapeHtmlUtf8(urls[0]);
 218
 219         URL = data.c_str();
 220         return LyricsFetcher::fetch("", "");
 221 }
 222
 223 bool GoogleLyricsFetcher::isURLOk(const std::string &url)
 224 {
 225         return url.find(siteKeyword()) != std::string::npos;
 226 }
 227
 228 /**********************************************************************/
 229
 230 bool MetrolyricsFetcher::isURLOk(const std::string &url)
 231 {
 232         // it sometimes return link to sitemap.xml, which is huge so we need to discard it
 233         return GoogleLyricsFetcher::isURLOk(url) && url.find("sitemap") == std::string::npos;
 234 }
 235
 236 /**********************************************************************/
 237
 238 LyricsFetcher::Result InternetLyricsFetcher::fetch(const std::string &artist, const std::string &title)
 239 {
 240         GoogleLyricsFetcher::fetch(artist, title);
 241         LyricsFetcher::Result result;
 242         result.first = false;
 243         result.second = "The following site may contain lyrics for this song: ";
 244         result.second += URL;
 245         return result;
 246 }
 247
 248 bool InternetLyricsFetcher::isURLOk(const std::string &url)
 249 {
 250         URL = url;
 251         return false;
 252 }
 253
 254 #endif // HAVE_CURL_CURL_H
 255