1 /***************************************************************************
2 * Copyright (C) 2008-2016 by Andrzej Rybczak *
3 * electricityispower@gmail.com *
5 * This program is free software; you can redistribute it and/or modify *
6 * it under the terms of the GNU General Public License as published by *
7 * the Free Software Foundation; either version 2 of the License, or *
8 * (at your option) any later version. *
10 * This program is distributed in the hope that it will be useful, *
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of *
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
13 * GNU General Public License for more details. *
15 * You should have received a copy of the GNU General Public License *
16 * along with this program; if not, write to the *
17 * Free Software Foundation, Inc., *
18 * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. *
19 ***************************************************************************/
22 #include "curl_handle.h"
26 #include <boost/algorithm/string/join.hpp>
27 #include <boost/algorithm/string/replace.hpp>
28 #include <boost/algorithm/string/split.hpp>
29 #include <boost/algorithm/string/trim.hpp>
30 #include <boost/regex.hpp>
33 #include "lyrics_fetcher.h"
34 #include "utility/html.h"
35 #include "utility/string.h"
37 std::istream
&operator>>(std::istream
&is
, LyricsFetcher_
&fetcher
)
42 fetcher
= std::make_unique
<LyricwikiFetcher
>();
43 else if (s
== "azlyrics")
44 fetcher
= std::make_unique
<AzLyricsFetcher
>();
45 else if (s
== "genius")
46 fetcher
= std::make_unique
<GeniusFetcher
>();
47 else if (s
== "sing365")
48 fetcher
= std::make_unique
<Sing365Fetcher
>();
49 else if (s
== "lyricsmania")
50 fetcher
= std::make_unique
<LyricsmaniaFetcher
>();
51 else if (s
== "metrolyrics")
52 fetcher
= std::make_unique
<MetrolyricsFetcher
>();
53 else if (s
== "justsomelyrics")
54 fetcher
= std::make_unique
<JustSomeLyricsFetcher
>();
55 else if (s
== "tekstowo")
56 fetcher
= std::make_unique
<TekstowoFetcher
>();
57 else if (s
== "internet")
58 fetcher
= std::make_unique
<InternetLyricsFetcher
>();
60 is
.setstate(std::ios::failbit
);
64 const char LyricsFetcher::msgNotFound
[] = "Not found";
66 LyricsFetcher::Result
LyricsFetcher::fetch(const std::string
&artist
, const std::string
&title
)
71 std::string url
= urlTemplate();
72 boost::replace_all(url
, "%artist%", artist
);
73 boost::replace_all(url
, "%title%", title
);
76 CURLcode code
= Curl::perform(data
, url
);
80 result
.second
= curl_easy_strerror(code
);
84 auto lyrics
= getContent(regex(), data
);
86 if (lyrics
.empty() || notLyrics(data
))
88 result
.second
= msgNotFound
;
93 for (auto it
= lyrics
.begin(); it
!= lyrics
.end(); ++it
)
99 if (it
!= lyrics
.end()-1)
100 data
+= "\n\n----------\n\n";
104 result
.second
= data
;
109 std::vector
<std::string
> LyricsFetcher::getContent(const char *regex_
, const std::string
&data
)
111 std::vector
<std::string
> result
;
112 boost::regex
rx(regex_
);
113 auto first
= boost::sregex_iterator(data
.begin(), data
.end(), rx
);
114 auto last
= boost::sregex_iterator();
115 for (; first
!= last
; ++first
)
116 result
.push_back(first
->str(1));
120 void LyricsFetcher::postProcess(std::string
&data
) const
122 data
= unescapeHtmlUtf8(data
);
124 // Remove indentation from each line and collapse multiple newlines into one.
125 std::vector
<std::string
> lines
;
126 boost::split(lines
, data
, boost::is_any_of("\n"));
127 for (auto &line
: lines
)
129 std::unique(lines
.begin(), lines
.end(), [](std::string
&a
, std::string
&b
) {
130 return a
.empty() && b
.empty();
132 data
= boost::algorithm::join(lines
, "\n");
136 /***********************************************************************/
138 LyricsFetcher::Result
LyricwikiFetcher::fetch(const std::string
&artist
, const std::string
&title
)
140 LyricsFetcher::Result result
= LyricsFetcher::fetch(artist
, title
);
141 if (result
.first
== true)
143 result
.first
= false;
146 CURLcode code
= Curl::perform(data
, result
.second
, "", true);
148 if (code
!= CURLE_OK
)
150 result
.second
= curl_easy_strerror(code
);
154 auto lyrics
= getContent("<div class='lyricbox'>(.*?)</div>", data
);
158 result
.second
= msgNotFound
;
161 std::transform(lyrics
.begin(), lyrics
.end(), lyrics
.begin(), unescapeHtmlUtf8
);
162 bool license_restriction
= std::any_of(lyrics
.begin(), lyrics
.end(), [](const std::string
&s
) {
163 return s
.find("Unfortunately, we are not licensed to display the full lyrics for this song at the moment.") != std::string::npos
;
165 if (license_restriction
)
167 result
.second
= "Licence restriction";
172 for (auto it
= lyrics
.begin(); it
!= lyrics
.end(); ++it
)
179 if (it
!= lyrics
.end()-1)
180 data
+= "\n\n----------\n\n";
184 result
.second
= data
;
190 bool LyricwikiFetcher::notLyrics(const std::string
&data
) const
192 return data
.find("action=edit") != std::string::npos
;
195 /**********************************************************************/
197 LyricsFetcher::Result
GoogleLyricsFetcher::fetch(const std::string
&artist
, const std::string
&title
)
200 result
.first
= false;
202 std::string search_str
= artist
;
205 search_str
+= "+%2B";
206 search_str
+= siteKeyword();
208 std::string google_url
= "http://www.google.com/search?hl=en&ie=UTF-8&oe=UTF-8&q=";
209 google_url
+= search_str
;
210 google_url
+= "&btnI=I%27m+Feeling+Lucky";
213 CURLcode code
= Curl::perform(data
, google_url
, google_url
);
215 if (code
!= CURLE_OK
)
217 result
.second
= curl_easy_strerror(code
);
221 auto urls
= getContent("<A HREF=\"(.*?)\">here</A>", data
);
223 if (urls
.empty() || !isURLOk(urls
[0]))
225 result
.second
= msgNotFound
;
229 data
= unescapeHtmlUtf8(urls
[0]);
232 return LyricsFetcher::fetch("", "");
235 bool GoogleLyricsFetcher::isURLOk(const std::string
&url
)
237 return url
.find(siteKeyword()) != std::string::npos
;
240 /**********************************************************************/
242 bool MetrolyricsFetcher::isURLOk(const std::string
&url
)
244 // it sometimes return link to sitemap.xml, which is huge so we need to discard it
245 return GoogleLyricsFetcher::isURLOk(url
) && url
.find("sitemap") == std::string::npos
;
248 /**********************************************************************/
250 LyricsFetcher::Result
InternetLyricsFetcher::fetch(const std::string
&artist
, const std::string
&title
)
252 GoogleLyricsFetcher::fetch(artist
, title
);
253 LyricsFetcher::Result result
;
254 result
.first
= false;
255 result
.second
= "The following site may contain lyrics for this song: ";
256 result
.second
+= URL
;
260 bool InternetLyricsFetcher::isURLOk(const std::string
&url
)