Bug 13758: Correct KOHA::VERSION in OverDrive.pm
[koha.git] / tools / export.pl
blobf63e2c4f6f55432a31a5db9b9c377b69cff9c938
1 #!/usr/bin/perl
4 # This file is part of Koha.
6 # Koha is free software; you can redistribute it and/or modify it
7 # under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
11 # Koha is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with Koha; if not, see <http://www.gnu.org/licenses>.
19 use Modern::Perl;
20 use MARC::File::XML;
21 use List::MoreUtils qw(uniq);
22 use Getopt::Long;
23 use CGI qw ( -utf8 );
24 use C4::Auth;
25 use C4::AuthoritiesMarc; # GetAuthority
26 use C4::Biblio; # GetMarcBiblio
27 use C4::Branch; # GetBranches
28 use C4::Csv;
29 use C4::Koha; # GetItemTypes
30 use C4::Output;
31 use C4::Record;
33 my $query = new CGI;
35 my $clean;
36 my $dont_export_items;
37 my $deleted_barcodes;
38 my $timestamp;
39 my $record_type;
40 my $id_list_file;
41 my $help;
42 my $op = $query->param("op") || '';
43 my $filename = $query->param("filename") || 'koha.mrc';
44 my $dbh = C4::Context->dbh;
45 my $marcflavour = C4::Context->preference("marcflavour");
46 my $output_format = $query->param("format") || $query->param("output_format") || 'iso2709';
48 # Checks if the script is called from commandline
49 my $commandline = not defined $ENV{GATEWAY_INTERFACE};
52 # @biblionumbers is only use for csv export from circulation.pl
53 my @biblionumbers = uniq $query->param("biblionumbers");
55 if ( $commandline ) {
57 # Getting parameters
58 $op = 'export';
59 GetOptions(
60 'format=s' => \$output_format,
61 'date=s' => \$timestamp,
62 'dont_export_items' => \$dont_export_items,
63 'deleted_barcodes' => \$deleted_barcodes,
64 'clean' => \$clean,
65 'filename=s' => \$filename,
66 'record-type=s' => \$record_type,
67 'id_list_file=s' => \$id_list_file,
68 'help|?' => \$help
71 if ($help) {
72 print <<_USAGE_;
73 export.pl [--format=format] [--date=date] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
76 --format=FORMAT FORMAT is either 'xml' or 'marc' (default)
78 --date=DATE DATE should be entered as the 'dateformat' syspref is
79 set (dd/mm/yyyy for metric, yyyy-mm-dd for iso,
80 mm/dd/yyyy for us) records exported are the ones that
81 have been modified since DATE
83 --record-type=TYPE TYPE is 'bibs' or 'auths'
85 --deleted_barcodes If used, a list of barcodes of items deleted since DATE
86 is produced (or from all deleted items if no date is
87 specified). Used only if TYPE is 'bibs'
89 --clean removes NSE/NSB
91 --id_list_file=PATH PATH is a path to a file containing a list of
92 IDs (biblionumber or authid) with one ID per line.
93 This list works as a filter; it is compatible with
94 other parameters for selecting records
95 _USAGE_
96 exit;
99 # Default parameters values :
100 $timestamp ||= '';
101 $dont_export_items ||= 0;
102 $deleted_barcodes ||= 0;
103 $clean ||= 0;
104 $record_type ||= "bibs";
105 $id_list_file ||= 0;
107 # Redirect stdout
108 open STDOUT, '>', $filename if $filename;
111 else {
113 $op = $query->param("op") || '';
114 $filename = $query->param("filename") || 'koha.mrc';
115 $filename =~ s/(\r|\n)//;
119 # Default value for output_format is 'iso2709'
120 $output_format ||= 'iso2709';
121 # Retrocompatibility for the format parameter
122 $output_format = 'iso2709' if $output_format eq 'marc';
124 my ( $template, $loggedinuser, $cookie, $flags ) = get_template_and_user(
126 template_name => "tools/export.tt",
127 query => $query,
128 type => "intranet",
129 authnotrequired => $commandline,
130 flagsrequired => { tools => 'export_catalog' },
131 debug => 1,
135 my $limit_ind_branch =
136 ( C4::Context->preference('IndependentBranches')
137 && C4::Context->userenv
138 && !C4::Context->IsSuperLibrarian()
139 && C4::Context->userenv->{branch} ) ? 1 : 0;
141 my @branch = $query->param("branch");
142 if ( C4::Context->preference("IndependentBranches")
143 && C4::Context->userenv
144 && !C4::Context->IsSuperLibrarian() )
146 @branch = ( C4::Context->userenv->{'branch'} );
148 # if stripping nonlocal items, use loggedinuser's branch
149 my $localbranch = C4::Context->userenv->{'branch'};
151 my %branchmap = map { $_ => 1 } @branch; # for quick lookups
153 my $backupdir = C4::Context->config('backupdir');
155 if ( $op eq "export" ) {
156 if (
157 $output_format eq "iso2709"
158 or $output_format eq "xml"
159 or (
160 $output_format eq 'csv'
161 and not @biblionumbers
164 my $charset = 'utf-8';
165 my $mimetype = 'application/octet-stream';
167 binmode STDOUT, ':encoding(UTF-8)'
168 if $filename =~ m/\.gz$/
169 or $filename =~ m/\.bz2$/;
171 if ( $filename =~ m/\.gz$/ ) {
172 $mimetype = 'application/x-gzip';
173 $charset = '';
174 binmode STDOUT;
176 elsif ( $filename =~ m/\.bz2$/ ) {
177 $mimetype = 'application/x-bzip2';
178 binmode STDOUT;
179 $charset = '';
181 print $query->header(
182 -type => $mimetype,
183 -charset => $charset,
184 -attachment => $filename,
185 ) unless ($commandline);
187 $record_type = $query->param("record_type") unless ($commandline);
188 my $export_remove_fields = $query->param("export_remove_fields");
189 my @biblionumbers = $query->param("biblionumbers");
190 my @itemnumbers = $query->param("itemnumbers");
191 my @sql_params;
192 my $sql_query;
193 my @recordids;
195 my $StartingBiblionumber = $query->param("StartingBiblionumber");
196 my $EndingBiblionumber = $query->param("EndingBiblionumber");
197 my $itemtype = $query->param("itemtype");
198 my $start_callnumber = $query->param("start_callnumber");
199 my $end_callnumber = $query->param("end_callnumber");
200 $timestamp = ($timestamp) ? C4::Dates->new($timestamp) : ''
201 if ($commandline);
202 my $start_accession =
203 ( $query->param("start_accession") )
204 ? C4::Dates->new( $query->param("start_accession") )
205 : '';
206 my $end_accession =
207 ( $query->param("end_accession") )
208 ? C4::Dates->new( $query->param("end_accession") )
209 : '';
210 $dont_export_items = $query->param("dont_export_item")
211 unless ($commandline);
213 my $strip_nonlocal_items = $query->param("strip_nonlocal_items");
215 my $biblioitemstable =
216 ( $commandline and $deleted_barcodes )
217 ? 'deletedbiblioitems'
218 : 'biblioitems';
219 my $itemstable =
220 ( $commandline and $deleted_barcodes )
221 ? 'deleteditems'
222 : 'items';
224 my $starting_authid = $query->param('starting_authid');
225 my $ending_authid = $query->param('ending_authid');
226 my $authtype = $query->param('authtype');
227 my $filefh;
228 if ($commandline) {
229 open $filefh,"<", $id_list_file or die "cannot open $id_list_file: $!" if $id_list_file;
230 } else {
231 $filefh = $query->upload("id_list_file");
233 my %id_filter;
234 if ($filefh) {
235 while (my $number=<$filefh>){
236 $number=~s/[\r\n]*$//;
237 $id_filter{$number}=1 if $number=~/^\d+$/;
241 if ( $record_type eq 'bibs' and not @biblionumbers ) {
242 if ($timestamp) {
244 # Specific query when timestamp is used
245 # Actually it's used only with CLI and so all previous filters
246 # are not used.
247 # If one day timestamp is used via the web interface, this part will
248 # certainly have to be rewrited
249 my ( $query, $params ) = construct_query(
251 recordtype => $record_type,
252 timestamp => $timestamp,
253 biblioitemstable => $biblioitemstable,
256 $sql_query = $query;
257 @sql_params = @$params;
260 else {
261 my ( $query, $params ) = construct_query(
263 recordtype => $record_type,
264 biblioitemstable => $biblioitemstable,
265 itemstable => $itemstable,
266 StartingBiblionumber => $StartingBiblionumber,
267 EndingBiblionumber => $EndingBiblionumber,
268 branch => \@branch,
269 start_callnumber => $start_callnumber,
270 end_callnumber => $end_callnumber,
271 start_accession => $start_accession,
272 end_accession => $end_accession,
273 itemtype => $itemtype,
276 $sql_query = $query;
277 @sql_params = @$params;
280 elsif ( $record_type eq 'auths' ) {
281 my ( $query, $params ) = construct_query(
283 recordtype => $record_type,
284 starting_authid => $starting_authid,
285 ending_authid => $ending_authid,
286 authtype => $authtype,
289 $sql_query = $query;
290 @sql_params = @$params;
293 elsif ( $record_type eq 'db' ) {
294 my $successful_export;
295 if ( $flags->{superlibrarian}
296 && C4::Context->config('backup_db_via_tools') )
298 $successful_export = download_backup(
300 directory => "$backupdir",
301 extension => 'sql',
302 filename => "$filename"
306 unless ($successful_export) {
307 my $remotehost = $query->remote_host();
308 $remotehost =~ s/(\n|\r)//;
309 warn
310 "A suspicious attempt was made to download the db at '$filename' by someone at "
311 . $remotehost . "\n";
313 exit;
315 elsif ( $record_type eq 'conf' ) {
316 my $successful_export;
317 if ( $flags->{superlibrarian}
318 && C4::Context->config('backup_conf_via_tools') )
320 $successful_export = download_backup(
322 directory => "$backupdir",
323 extension => 'tar',
324 filename => "$filename"
328 unless ($successful_export) {
329 my $remotehost = $query->remote_host();
330 $remotehost =~ s/(\n|\r)//;
331 warn
332 "A suspicious attempt was made to download the configuration at '$filename' by someone at "
333 . $remotehost . "\n";
335 exit;
337 elsif (@biblionumbers) {
338 push @recordids, (@biblionumbers);
340 else {
342 # Someone is trying to mess us up
343 exit;
345 unless (@biblionumbers) {
346 my $sth = $dbh->prepare($sql_query);
347 $sth->execute(@sql_params);
348 push @recordids, map {
349 map { $$_[0] } $_
350 } @{ $sth->fetchall_arrayref };
351 @recordids = grep { exists($id_filter{$_}) } @recordids if scalar(%id_filter);
354 my $xml_header_written = 0;
355 for my $recordid ( uniq @recordids ) {
356 if ($deleted_barcodes) {
357 my $q = "
358 SELECT DISTINCT barcode
359 FROM deleteditems
360 WHERE deleteditems.biblionumber = ?
362 my $sth = $dbh->prepare($q);
363 $sth->execute($recordid);
364 while ( my $row = $sth->fetchrow_array ) {
365 print "$row\n";
368 else {
369 my $record;
370 if ( $record_type eq 'bibs' ) {
371 $record = eval { GetMarcBiblio($recordid); };
373 next if $@;
374 next if not defined $record;
375 C4::Biblio::EmbedItemsInMarcBiblio( $record, $recordid,
376 \@itemnumbers )
377 unless $dont_export_items;
378 if ( $strip_nonlocal_items
379 || $limit_ind_branch
380 || $dont_export_items )
382 my ( $homebranchfield, $homebranchsubfield ) =
383 GetMarcFromKohaField( 'items.homebranch', '' );
384 for my $itemfield ( $record->field($homebranchfield) ) {
385 $record->delete_field($itemfield)
386 if ( $dont_export_items
387 || $localbranch ne $itemfield->subfield(
388 $homebranchsubfield) );
392 elsif ( $record_type eq 'auths' ) {
393 $record = C4::AuthoritiesMarc::GetAuthority($recordid);
394 next if not defined $record;
397 if ($export_remove_fields) {
398 for my $f ( split / /, $export_remove_fields ) {
399 if ( $f =~ m/^(\d{3})(.)?$/ ) {
400 my ( $field, $subfield ) = ( $1, $2 );
402 # skip if this record doesn't have this field
403 if ( defined $record->field($field) ) {
404 if ( defined $subfield ) {
405 my @tags = $record->field($field);
406 foreach my $t (@tags) {
407 $t->delete_subfields($subfield);
410 else {
411 $record->delete_fields($record->field($field));
417 RemoveAllNsb($record) if ($clean);
418 if ( $output_format eq "xml" ) {
419 unless ($xml_header_written) {
420 MARC::File::XML->default_record_format(
422 $marcflavour eq 'UNIMARC'
423 && $record_type eq 'auths'
424 ) ? 'UNIMARCAUTH' : $marcflavour
426 print MARC::File::XML::header();
427 print "\n";
428 $xml_header_written = 1;
430 print MARC::File::XML::record($record);
431 print "\n";
433 elsif ( $output_format eq 'iso2709' ) {
434 my $errorcount_on_decode = eval { scalar(MARC::File::USMARC->decode( $record->as_usmarc )->warnings()) };
435 if ($errorcount_on_decode or $@){
436 warn $@ if $@;
437 warn "record (number $recordid) is invalid and therefore not exported because its reopening generates warnings above";
438 next;
440 print $record->as_usmarc();
444 if ($xml_header_written) {
445 print MARC::File::XML::footer();
446 print "\n";
448 if ( $output_format eq 'csv' ) {
449 my $csv_profile_id = $query->param('csv_profile')
450 || GetCsvProfileId( C4::Context->preference('ExportWithCsvProfile') );
451 my $output =
452 marc2csv( \@recordids,
453 $csv_profile_id );
455 print $output;
458 exit;
460 elsif ( $output_format eq "csv" ) {
461 my @biblionumbers = uniq $query->param("biblionumbers");
462 my @itemnumbers = $query->param("itemnumbers");
463 my $csv_profile_id = $query->param('csv_profile') || GetCsvProfileId( C4::Context->preference('ExportWithCsvProfile') );
464 my $output =
465 marc2csv( \@biblionumbers,
466 $csv_profile_id,
467 \@itemnumbers, );
468 print $query->header(
469 -type => 'application/octet-stream',
470 -'Content-Transfer-Encoding' => 'binary',
471 -attachment => "export.csv"
473 print $output;
474 exit;
476 } # if export
478 else {
480 my $itemtypes = GetItemTypes;
481 my @itemtypesloop;
482 foreach my $thisitemtype ( sort keys %$itemtypes ) {
483 my %row = (
484 value => $thisitemtype,
485 description => $itemtypes->{$thisitemtype}->{'description'},
487 push @itemtypesloop, \%row;
489 my $branches = GetBranches($limit_ind_branch);
490 my @branchloop;
491 for my $thisbranch (
492 sort { $branches->{$a}->{branchname} cmp $branches->{$b}->{branchname} }
493 keys %{$branches}
496 push @branchloop,
498 value => $thisbranch,
499 selected => %branchmap ? $branchmap{$thisbranch} : 1,
500 branchname => $branches->{$thisbranch}->{'branchname'},
504 my $authtypes = getauthtypes;
505 my @authtypesloop;
506 foreach my $thisauthtype ( sort keys %$authtypes ) {
507 next unless $thisauthtype;
508 my %row = (
509 value => $thisauthtype,
510 description => $authtypes->{$thisauthtype}->{'authtypetext'},
512 push @authtypesloop, \%row;
515 if ( $flags->{superlibrarian}
516 && C4::Context->config('backup_db_via_tools')
517 && $backupdir
518 && -d $backupdir )
520 $template->{VARS}->{'allow_db_export'} = 1;
521 $template->{VARS}->{'dbfiles'} = getbackupfilelist(
522 { directory => "$backupdir", extension => 'sql' } );
525 if ( $flags->{superlibrarian}
526 && C4::Context->config('backup_conf_via_tools')
527 && $backupdir
528 && -d $backupdir )
530 $template->{VARS}->{'allow_conf_export'} = 1;
531 $template->{VARS}->{'conffiles'} = getbackupfilelist(
532 { directory => "$backupdir", extension => 'tar' } );
535 $template->param(
536 branchloop => \@branchloop,
537 itemtypeloop => \@itemtypesloop,
538 authtypeloop => \@authtypesloop,
539 export_remove_fields => C4::Context->preference("ExportRemoveFields"),
540 csv_profiles => C4::Csv::GetCsvProfiles('marc'),
543 output_html_with_http_headers $query, $cookie, $template->output;
546 sub construct_query {
547 my ($params) = @_;
549 my ( $sql_query, @sql_params );
551 if ( $params->{recordtype} eq "bibs" ) {
552 if ( $params->{timestamp} ) {
553 my $biblioitemstable = $params->{biblioitemstable};
554 $sql_query = " (
555 SELECT biblionumber
556 FROM $biblioitemstable
557 LEFT JOIN items USING(biblionumber)
558 WHERE $biblioitemstable.timestamp >= ?
559 OR items.timestamp >= ?
560 ) UNION (
561 SELECT biblionumber
562 FROM $biblioitemstable
563 LEFT JOIN deleteditems USING(biblionumber)
564 WHERE $biblioitemstable.timestamp >= ?
565 OR deleteditems.timestamp >= ?
566 ) ";
567 my $ts = $timestamp->output('iso');
568 @sql_params = ( $ts, $ts, $ts, $ts );
570 else {
571 my $biblioitemstable = $params->{biblioitemstable};
572 my $itemstable = $params->{itemstable};
573 my $StartingBiblionumber = $params->{StartingBiblionumber};
574 my $EndingBiblionumber = $params->{EndingBiblionumber};
575 my @branch = @{ $params->{branch} };
576 my $start_callnumber = $params->{start_callnumber};
577 my $end_callnumber = $params->{end_callnumber};
578 my $start_accession = $params->{start_accession};
579 my $end_accession = $params->{end_accession};
580 my $itemtype = $params->{itemtype};
581 my $items_filter =
582 @branch
583 || $start_callnumber
584 || $end_callnumber
585 || $start_accession
586 || $end_accession
587 || ( $itemtype && C4::Context->preference('item-level_itypes') );
588 $sql_query = $items_filter
589 ? "SELECT DISTINCT $biblioitemstable.biblionumber
590 FROM $biblioitemstable JOIN $itemstable
591 USING (biblionumber) WHERE 1"
592 : "SELECT $biblioitemstable.biblionumber FROM $biblioitemstable WHERE biblionumber >0 ";
594 if ($StartingBiblionumber) {
595 $sql_query .= " AND $biblioitemstable.biblionumber >= ? ";
596 push @sql_params, $StartingBiblionumber;
599 if ($EndingBiblionumber) {
600 $sql_query .= " AND $biblioitemstable.biblionumber <= ? ";
601 push @sql_params, $EndingBiblionumber;
604 if (@branch) {
605 $sql_query .= " AND homebranch IN (".join(',',map({'?'} @branch)).")";
606 push @sql_params, @branch;
609 if ($start_callnumber) {
610 $sql_query .= " AND itemcallnumber >= ? ";
611 push @sql_params, $start_callnumber;
614 if ($end_callnumber) {
615 $sql_query .= " AND itemcallnumber <= ? ";
616 push @sql_params, $end_callnumber;
618 if ($start_accession) {
619 $sql_query .= " AND dateaccessioned >= ? ";
620 push @sql_params, $start_accession->output('iso');
623 if ($end_accession) {
624 $sql_query .= " AND dateaccessioned <= ? ";
625 push @sql_params, $end_accession->output('iso');
628 if ($itemtype) {
629 $sql_query .=
630 ( C4::Context->preference('item-level_itypes') )
631 ? " AND items.itype = ? "
632 : " AND biblioitems.itemtype = ?";
633 push @sql_params, $itemtype;
637 elsif ( $params->{recordtype} eq "auths" ) {
638 if ( $params->{timestamp} ) {
640 #TODO
642 else {
643 my $starting_authid = $params->{starting_authid};
644 my $ending_authid = $params->{ending_authid};
645 my $authtype = $params->{authtype};
646 $sql_query =
647 "SELECT DISTINCT auth_header.authid FROM auth_header WHERE 1";
649 if ($starting_authid) {
650 $sql_query .= " AND auth_header.authid >= ? ";
651 push @sql_params, $starting_authid;
654 if ($ending_authid) {
655 $sql_query .= " AND auth_header.authid <= ? ";
656 push @sql_params, $ending_authid;
659 if ($authtype) {
660 $sql_query .= " AND auth_header.authtypecode = ? ";
661 push @sql_params, $authtype;
665 return ( $sql_query, \@sql_params );
668 sub getbackupfilelist {
669 my $args = shift;
670 my $directory = $args->{directory};
671 my $extension = $args->{extension};
672 my @files;
674 if ( opendir( my $dir, $directory ) ) {
675 while ( my $file = readdir($dir) ) {
676 next unless ( $file =~ m/\.$extension(\.(gz|bz2|xz))?/ );
677 push @files, $file
678 if ( -f "$directory/$file" && -r "$directory/$file" );
680 closedir($dir);
682 return \@files;
685 sub download_backup {
686 my $args = shift;
687 my $directory = $args->{directory};
688 my $extension = $args->{extension};
689 my $filename = $args->{filename};
691 return unless ( $directory && -d $directory );
692 return unless ( $filename =~ m/\.$extension(\.(gz|bz2|xz))?$/ );
693 return if ( $filename =~ m#/# );
694 $filename = "$directory/$filename";
695 return unless ( -f $filename && -r $filename );
696 return unless ( open( my $dump, '<', $filename ) );
697 binmode $dump;
699 while ( read( $dump, my $data, 64 * 1024 ) ) {
700 print $data;
702 close($dump);
703 return 1;