Skip to content

Commit

Permalink
Format license headers
Browse files Browse the repository at this point in the history
  • Loading branch information
esotericpig committed Jun 25, 2021
1 parent 65d5c10 commit 54f29cb
Show file tree
Hide file tree
Showing 37 changed files with 139 additions and 524 deletions.
38 changes: 19 additions & 19 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -435,18 +435,18 @@ require 'nhkore/scraper'
s = NHKore::Scraper.new('https://www3.nhk.or.jp/news/easy/',
open_timeout: 300, # Open timeout in seconds (default: nil)
read_timeout: 300, # Read timeout in seconds (default: nil)
# Maximum number of times to retry the URL
# - default: 3
# - Open/connect will fail a couple of times on a bad/slow internet connection.
max_retries: 10,
# Maximum number of redirects allowed.
# - default: 3
# - You can set this to nil or -1, but I recommend using a number
# for safety (infinite-loop attack).
max_redirects: 1,
# How to check redirect URLs for safety.
# - default: :strict
# - nil => do not check
Expand All @@ -455,15 +455,15 @@ s = NHKore::Scraper.new('https://www3.nhk.or.jp/news/easy/',
# - :strict => check the scheme and domain
# (i.e., if https://bing.com, redirect URL must be https://bing.com)
redirect_rule: :lenient,
# Set the HTTP header field 'cookie' from the 'set-cookie' response.
# - default: false
# - Currently uses the 'http-cookie' Gem.
# - This is currently a time-consuming operation because it opens the URL twice.
# - Necessary for Search Engines or other sites that require cookies
# in order to block bots.
eat_cookie: true,
# Set HTTP header fields.
# - default: nil
# - Necessary for Search Engines or other sites that try to block bots.
Expand Down Expand Up @@ -526,9 +526,9 @@ doc = ss.html_doc()
doc.css('a').each() do |anchor|
link = anchor['href']
next if ss.ignore_link?(link,cleaned: false)
if link.include?('https://www3.nhk')
puts link
end
Expand All @@ -549,9 +549,9 @@ page_num = 1
while !next_page.empty?()
puts "Page #{page_num += 1}: #{next_page.count}"
bs = NHKore::BingScraper.new(:yasashii,url: next_page.url)
next_page = bs.scrape(slinks,next_page)
end
Expand All @@ -570,24 +570,24 @@ require 'time'
as = NHKore::ArticleScraper.new(
'https://www3.nhk.or.jp/news/easy/k10011862381000/k10011862381000.html',
# If false, scrape the article leniently (for older articles which
# may not have certain tags, etc.).
# - default: true
strict: false,
# {Dict} to use as the dictionary for words (Easy articles).
# - default: :scrape
# - nil => don't scrape/use it (necessary for Regular articles)
# - :scrape => auto-scrape it using {DictScraper}
# - {Dict} => your own {Dict}
dict: nil,
# Date time to use as a fallback if the article doesn't have one
# (for older articles).
# - default: nil
datetime: Time.new(2020,2,2),
# Year to use as a fallback if the article doesn't have one
# (for older articles).
# - default: nil
Expand Down Expand Up @@ -624,7 +624,7 @@ require 'nhkore/dict_scraper'
url = 'https://www3.nhk.or.jp/news/easy/k10011862381000/k10011862381000.html'
ds = NHKore::DictScraper.new(
url,
# Change the URL appropriately to the dictionary URL.
# - default: true
parse_url: true,
Expand All @@ -637,13 +637,13 @@ dict = ds.scrape()
dict.entries.each() do |key,entry|
entry.id
entry.defns.each() do |defn|
defn.hyoukis.each() {|hyouki| }
defn.text
defn.words.each() {|word| }
end
puts entry.build_hyouki()
puts entry.build_defn()
puts '---'
Expand Down Expand Up @@ -789,13 +789,13 @@ JPN = ['桜','ぶ','ブ']
def fmt_jpn()
fmt = []
JPN.each() do |x|
x = yield(x)
x = x ? "\u2B55" : Util::JPN_SPACE unless x.is_a?(String)
fmt << x
end
return "[ #{fmt.join(' | ')} ]"
end
Expand Down Expand Up @@ -879,7 +879,7 @@ Releasing new HTML file for website:
[GNU LGPL v3+](LICENSE.txt)
> NHKore (<https://github.com/esotericpig/nhkore>)
> Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
> Copyright (c) 2020-2021 Jonathan Bradley Whited
>
> NHKore is free software: you can redistribute it and/or modify
> it under the terms of the GNU Lesser General Public License as published by
Expand Down
15 changes: 2 additions & 13 deletions bin/nhkore
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,9 @@

#--
# This file is part of NHKore.
# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
# Copyright (c) 2020-2021 Jonathan Bradley Whited
#
# NHKore is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# NHKore is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with NHKore. If not, see <https://www.gnu.org/licenses/>.
# SPDX-License-Identifier: LGPL-3.0-or-later
#++


Expand Down
17 changes: 3 additions & 14 deletions lib/nhkore.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,9 @@

#--
# This file is part of NHKore.
# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
# Copyright (c) 2020-2021 Jonathan Bradley Whited
#
# NHKore is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# NHKore is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with NHKore. If not, see <https://www.gnu.org/licenses/>.
# SPDX-License-Identifier: LGPL-3.0-or-later
#++


Expand All @@ -38,7 +27,7 @@


###
# @author Jonathan Bradley Whited (@esotericpig)
# @author Jonathan Bradley Whited
# @since 0.1.0
###
module NHKore
Expand Down
23 changes: 6 additions & 17 deletions lib/nhkore/app.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,9 @@

#--
# This file is part of NHKore.
# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
# Copyright (c) 2020-2021 Jonathan Bradley Whited
#
# NHKore is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# NHKore is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with NHKore. If not, see <https://www.gnu.org/licenses/>.
# SPDX-License-Identifier: LGPL-3.0-or-later
#++


Expand All @@ -39,7 +28,7 @@

module NHKore
###
# @author Jonathan Bradley Whited (@esotericpig)
# @author Jonathan Bradley Whited
# @since 0.2.0
###
module CLI
Expand All @@ -48,7 +37,7 @@ module CLI
###
# For disabling/enabling color output.
#
# @author Jonathan Bradley Whited (@esotericpig)
# @author Jonathan Bradley Whited
# @since 0.2.1
###
module CriColorExt
Expand All @@ -64,7 +53,7 @@ def color?(io)
end

###
# @author Jonathan Bradley Whited (@esotericpig)
# @author Jonathan Bradley Whited
# @since 0.2.0
###
class App
Expand Down Expand Up @@ -578,7 +567,7 @@ def update_spin_detail(detail)
end

###
# @author Jonathan Bradley Whited (@esotericpig)
# @author Jonathan Bradley Whited
# @since 0.2.0
###
class NoProgressBar
Expand Down
17 changes: 3 additions & 14 deletions lib/nhkore/article.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,9 @@

#--
# This file is part of NHKore.
# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
# Copyright (c) 2020-2021 Jonathan Bradley Whited
#
# NHKore is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# NHKore is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with NHKore. If not, see <https://www.gnu.org/licenses/>.
# SPDX-License-Identifier: LGPL-3.0-or-later
#++


Expand All @@ -28,7 +17,7 @@

module NHKore
###
# @author Jonathan Bradley Whited (@esotericpig)
# @author Jonathan Bradley Whited
# @since 0.2.0
###
class Article
Expand Down
19 changes: 4 additions & 15 deletions lib/nhkore/article_scraper.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,9 @@

#--
# This file is part of NHKore.
# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
# Copyright (c) 2020-2021 Jonathan Bradley Whited
#
# NHKore is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# NHKore is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with NHKore. If not, see <https://www.gnu.org/licenses/>.
# SPDX-License-Identifier: LGPL-3.0-or-later
#++


Expand All @@ -38,7 +27,7 @@

module NHKore
###
# @author Jonathan Bradley Whited (@esotericpig)
# @author Jonathan Bradley Whited
# @since 0.2.0
###
class ArticleScraper < Scraper
Expand Down Expand Up @@ -627,7 +616,7 @@ def warn_or_error(klass,msg)
end

###
# @author Jonathan Bradley Whited (@esotericpig)
# @author Jonathan Bradley Whited
# @since 0.2.0
###
class ScrapeWordsResult
Expand Down
21 changes: 5 additions & 16 deletions lib/nhkore/cleaner.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,9 @@

#--
# This file is part of NHKore.
# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
# Copyright (c) 2020-2021 Jonathan Bradley Whited
#
# NHKore is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# NHKore is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with NHKore. If not, see <https://www.gnu.org/licenses/>.
# SPDX-License-Identifier: LGPL-3.0-or-later
#++


Expand All @@ -26,7 +15,7 @@

module NHKore
###
# @author Jonathan Bradley Whited (@esotericpig)
# @author Jonathan Bradley Whited
# @since 0.2.0
###
class Cleaner
Expand Down Expand Up @@ -65,7 +54,7 @@ def self.clean_any(obj,cleaners)
end

###
# @author Jonathan Bradley Whited (@esotericpig)
# @author Jonathan Bradley Whited
# @since 0.2.0
###
class BasicCleaner < Cleaner
Expand All @@ -82,7 +71,7 @@ def end_clean(str)
end

###
# @author Jonathan Bradley Whited (@esotericpig)
# @author Jonathan Bradley Whited
# @since 0.2.0
###
class BestCleaner < BasicCleaner
Expand Down
Loading

0 comments on commit 54f29cb

Please sign in to comment.