src/lyrics_fetcher.cpp

   1 /***************************************************************************
   2  *   Copyright (C) 2008-2014 by Andrzej Rybczak                            *
   3  *   electricityispower@gmail.com                                          *
   4  *                                                                         *
   5  *   This program is free software; you can redistribute it and/or modify  *
   6  *   it under the terms of the GNU General Public License as published by  *
   7  *   the Free Software Foundation; either version 2 of the License, or     *
   8  *   (at your option) any later version.                                   *
   9  *                                                                         *
  10  *   This program is distributed in the hope that it will be useful,       *
  11  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  12  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  13  *   GNU General Public License for more details.                          *
  14  *                                                                         *
  15  *   You should have received a copy of the GNU General Public License     *
  16  *   along with this program; if not, write to the                         *
  17  *   Free Software Foundation, Inc.,                                       *
  18  *   51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.              *
  19  ***************************************************************************/
  20
  21 #include "config.h"
  22 #include "curl_handle.h"
  23
  24 #ifdef HAVE_CURL_CURL_H
  25
  26 #include <cstdlib>
  27 #include <cstring>
  28 #include <boost/algorithm/string/replace.hpp>
  29 #include <boost/algorithm/string/trim.hpp>
  30 #include <boost/regex.hpp>
  31
  32 #include "charset.h"
  33 #include "lyrics_fetcher.h"
  34 #include "utility/html.h"
  35 #include "utility/string.h"
  36
  37 LyricsFetcher *lyricsPlugins[] =
  38 {
  39         new LyricwikiFetcher(),
  40         new AzLyricsFetcher(),
  41         new Sing365Fetcher(),
  42         new LyricsmaniaFetcher(),
  43         new MetrolyricsFetcher(),
  44         new JustSomeLyricsFetcher(),
  45         new InternetLyricsFetcher(),
  46         0
  47 };
  48
  49 const char LyricsFetcher::msgNotFound[] = "Not found";
  50
  51 LyricsFetcher::Result LyricsFetcher::fetch(const std::string &artist, const std::string &title)
  52 {
  53         Result result;
  54         result.first = false;
  55
  56         std::string url = urlTemplate();
  57         boost::replace_all(url, "%artist%", artist);
  58         boost::replace_all(url, "%title%", title);
  59
  60         std::string data;
  61         CURLcode code = Curl::perform(data, url);
  62
  63         if (code != CURLE_OK)
  64         {
  65                 result.second = curl_easy_strerror(code);
  66                 return result;
  67         }
  68
  69         auto lyrics = getContent(regex(), data);
  70
  71         if (lyrics.empty() || notLyrics(data))
  72         {
  73                 result.second = msgNotFound;
  74                 return result;
  75         }
  76
  77         data.clear();
  78         for (auto it = lyrics.begin(); it != lyrics.end(); ++it)
  79         {
  80                 postProcess(*it);
  81                 if (!it->empty())
  82                 {
  83                         data += *it;
  84                         if (it != lyrics.end()-1)
  85                                 data += "\n\n----------\n\n";
  86                 }
  87         }
  88
  89         result.second = data;
  90         result.first = true;
  91         return result;
  92 }
  93
  94 std::vector<std::string> LyricsFetcher::getContent(const char *regex_, const std::string &data)
  95 {
  96         std::vector<std::string> result;
  97         boost::regex rx(regex_);
  98         auto first = boost::sregex_iterator(data.begin(), data.end(), rx);
  99         auto last = boost::sregex_iterator();
 100         for (; first != last; ++first)
 101                 result.push_back(first->str(1));
 102         return result;
 103 }
 104
 105 void LyricsFetcher::postProcess(std::string &data)
 106 {
 107         stripHtmlTags(data);
 108         boost::trim(data);
 109 }
 110
 111 /***********************************************************************/
 112
 113 LyricsFetcher::Result LyricwikiFetcher::fetch(const std::string &artist, const std::string &title)
 114 {
 115         LyricsFetcher::Result result = LyricsFetcher::fetch(artist, title);
 116         if (result.first == true)
 117         {
 118                 result.first = false;
 119
 120                 std::string data;
 121                 CURLcode code = Curl::perform(data, result.second);
 122
 123                 if (code != CURLE_OK)
 124                 {
 125                         result.second = curl_easy_strerror(code);
 126                         return result;
 127                 }
 128
 129                 auto lyrics = getContent("<div class='lyricbox'><script>.*?</script>(.*?)<!--", data);
 130
 131                 if (lyrics.empty())
 132                 {
 133                         result.second = msgNotFound;
 134                         return result;
 135                 }
 136                 std::transform(lyrics.begin(), lyrics.end(), lyrics.begin(), unescapeHtmlUtf8);
 137                 bool license_restriction = std::any_of(lyrics.begin(), lyrics.end(), [](const std::string &s) {
 138                         return s.find("Unfortunately, we are not licensed to display the full lyrics for this song at the moment.") != std::string::npos;
 139                 });
 140                 if (license_restriction)
 141                 {
 142                         result.second = "Licence restriction";
 143                         return result;
 144                 }
 145
 146                 data.clear();
 147                 for (auto it = lyrics.begin(); it != lyrics.end(); ++it)
 148                 {
 149                         boost::replace_all(*it, "<br />", "\n");
 150                         stripHtmlTags(*it);
 151                         boost::trim(*it);
 152                         if (!it->empty())
 153                         {
 154                                 data += *it;
 155                                 if (it != lyrics.end()-1)
 156                                         data += "\n\n----------\n\n";
 157                         }
 158                 }
 159
 160                 result.second = data;
 161                 result.first = true;
 162         }
 163         return result;
 164 }
 165
 166 bool LyricwikiFetcher::notLyrics(const std::string &data)
 167 {
 168         return data.find("action=edit") != std::string::npos;
 169 }
 170
 171 /**********************************************************************/
 172
 173 LyricsFetcher::Result GoogleLyricsFetcher::fetch(const std::string &artist, const std::string &title)
 174 {
 175         Result result;
 176         result.first = false;
 177
 178         std::string search_str = artist;
 179         search_str += "+";
 180         search_str += title;
 181         search_str += "+%2B";
 182         search_str += siteKeyword();
 183
 184         std::string google_url = "http://www.google.com/search?hl=en&ie=UTF-8&oe=UTF-8&q=";
 185         google_url += search_str;
 186         google_url += "&btnI=I%27m+Feeling+Lucky";
 187
 188         std::string data;
 189         CURLcode code = Curl::perform(data, google_url, google_url);
 190
 191         if (code != CURLE_OK)
 192         {
 193                 result.second = curl_easy_strerror(code);
 194                 return result;
 195         }
 196
 197         auto urls = getContent("<A HREF=\"(.*?)\">here</A>", data);
 198
 199         if (urls.empty() || !isURLOk(urls[0]))
 200         {
 201                 result.second = msgNotFound;
 202                 return result;
 203         }
 204
 205         data = unescapeHtmlUtf8(urls[0]);
 206
 207         URL = data.c_str();
 208         return LyricsFetcher::fetch("", "");
 209 }
 210
 211 bool GoogleLyricsFetcher::isURLOk(const std::string &url)
 212 {
 213         return url.find(siteKeyword()) != std::string::npos;
 214 }
 215
 216 /**********************************************************************/
 217
 218 void Sing365Fetcher::postProcess(std::string &data)
 219 {
 220         // throw away ad
 221         data = boost::regex_replace(data, boost::regex("<div.*</div>"), "");
 222         LyricsFetcher::postProcess(data);
 223 }
 224
 225 /**********************************************************************/
 226
 227 void MetrolyricsFetcher::postProcess(std::string &data)
 228 {
 229         // some of lyrics have both \n chars and <br />, html tags
 230         // are always present whereas \n chars are not, so we need to
 231         // throw them away to avoid having line breaks doubled.
 232         boost::replace_all(data, "&#10;", "");
 233         boost::replace_all(data, "<br />", "\n");
 234         data = unescapeHtmlUtf8(data);
 235         LyricsFetcher::postProcess(data);
 236 }
 237
 238 bool MetrolyricsFetcher::isURLOk(const std::string &url)
 239 {
 240         // it sometimes return link to sitemap.xml, which is huge so we need to discard it
 241         return GoogleLyricsFetcher::isURLOk(url) && url.find("sitemap") == std::string::npos;
 242 }
 243
 244 /**********************************************************************/
 245
 246 LyricsFetcher::Result InternetLyricsFetcher::fetch(const std::string &artist, const std::string &title)
 247 {
 248         GoogleLyricsFetcher::fetch(artist, title);
 249         LyricsFetcher::Result result;
 250         result.first = false;
 251         result.second = "The following site may contain lyrics for this song: ";
 252         result.second += URL;
 253         return result;
 254 }
 255
 256 bool InternetLyricsFetcher::isURLOk(const std::string &url)
 257 {
 258         URL = url;
 259         return false;
 260 }
 261
 262 #endif // HAVE_CURL_CURL_H
 263