Bug 11068: Update MARC21 es-ES default frameworks fields and translation
[koha.git] / tools / export.pl
blob97415f8d0f3d3554497d5a15ca2de7a465cbd760
1 #!/usr/bin/perl
4 # This file is part of Koha.
6 # Koha is free software; you can redistribute it and/or modify it under the
7 # terms of the GNU General Public License as published by the Free Software
8 # Foundation; either version 2 of the License, or (at your option) any later
9 # version.
11 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
12 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License along with
16 # Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
17 # Suite 330, Boston, MA 02111-1307 USA
19 use Modern::Perl;
20 use MARC::File::XML;
21 use List::MoreUtils qw(uniq);
22 use Getopt::Long;
23 use CGI;
24 use C4::Auth;
25 use C4::AuthoritiesMarc; # GetAuthority
26 use C4::Biblio; # GetMarcBiblio
27 use C4::Branch; # GetBranches
28 use C4::Csv;
29 use C4::Koha; # GetItemTypes
30 use C4::Output;
31 use C4::Record;
33 my $query = new CGI;
35 my $clean;
36 my $output_format;
37 my $dont_export_items;
38 my $deleted_barcodes;
39 my $timestamp;
40 my $record_type;
41 my $id_list_file;
42 my $help;
43 my $op = $query->param("op") || '';
44 my $filename = $query->param("filename") || 'koha.mrc';
45 my $dbh = C4::Context->dbh;
46 my $marcflavour = C4::Context->preference("marcflavour");
47 my $format = $query->param("format") || 'iso2709';
49 # Checks if the script is called from commandline
50 my $commandline = not defined $ENV{GATEWAY_INTERFACE};
52 if ( $commandline ) {
54 # Getting parameters
55 $op = 'export';
56 GetOptions(
57 'format=s' => \$output_format,
58 'date=s' => \$timestamp,
59 'dont_export_items' => \$dont_export_items,
60 'deleted_barcodes' => \$deleted_barcodes,
61 'clean' => \$clean,
62 'filename=s' => \$filename,
63 'record-type=s' => \$record_type,
64 'id_list_file=s' => \$id_list_file,
65 'help|?' => \$help
68 if ($help) {
69 print <<_USAGE_;
70 export.pl [--format=format] [--date=date] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
73 --format=FORMAT FORMAT is either 'xml' or 'marc' (default)
75 --date=DATE DATE should be entered as the 'dateformat' syspref is
76 set (dd/mm/yyyy for metric, yyyy-mm-dd for iso,
77 mm/dd/yyyy for us) records exported are the ones that
78 have been modified since DATE
80 --record-type=TYPE TYPE is 'bibs' or 'auths'
82 --deleted_barcodes If used, a list of barcodes of items deleted since DATE
83 is produced (or from all deleted items if no date is
84 specified). Used only if TYPE is 'bibs'
86 --clean removes NSE/NSB
88 --id_list_file=PATH PATH is a path to a file containing a list of
89 IDs (biblionumber or authid) with one ID per line.
90 This list works as a filter; it is compatible with
91 other parameters for selecting records
92 _USAGE_
93 exit;
96 # Default parameters values :
97 $output_format ||= 'marc';
98 $timestamp ||= '';
99 $dont_export_items ||= 0;
100 $deleted_barcodes ||= 0;
101 $clean ||= 0;
102 $record_type ||= "bibs";
103 $id_list_file ||= 0;
105 # Redirect stdout
106 open STDOUT, '>', $filename if $filename;
109 else {
111 $op = $query->param("op") || '';
112 $filename = $query->param("filename") || 'koha.mrc';
113 $filename =~ s/(\r|\n)//;
117 my ( $template, $loggedinuser, $cookie, $flags ) = get_template_and_user(
119 template_name => "tools/export.tt",
120 query => $query,
121 type => "intranet",
122 authnotrequired => $commandline,
123 flagsrequired => { tools => 'export_catalog' },
124 debug => 1,
128 my $limit_ind_branch =
129 ( C4::Context->preference('IndependentBranches')
130 && C4::Context->userenv
131 && !C4::Context->IsSuperLibrarian()
132 && C4::Context->userenv->{branch} ) ? 1 : 0;
134 my $branch = $query->param("branch") || '';
135 if ( C4::Context->preference("IndependentBranches")
136 && C4::Context->userenv
137 && !C4::Context->IsSuperLibrarian() )
139 $branch = C4::Context->userenv->{'branch'};
142 my $backupdir = C4::Context->config('backupdir');
144 if ( $op eq "export" ) {
145 if ( $format eq "iso2709" or $format eq "xml" ) {
146 my $charset = 'utf-8';
147 my $mimetype = 'application/octet-stream';
148 binmode STDOUT, ':encoding(UTF-8)';
149 if ( $filename =~ m/\.gz$/ ) {
150 $mimetype = 'application/x-gzip';
151 $charset = '';
152 binmode STDOUT;
154 elsif ( $filename =~ m/\.bz2$/ ) {
155 $mimetype = 'application/x-bzip2';
156 binmode STDOUT;
157 $charset = '';
159 print $query->header(
160 -type => $mimetype,
161 -charset => $charset,
162 -attachment => $filename
163 ) unless ($commandline);
165 $record_type = $query->param("record_type") unless ($commandline);
166 $output_format = $query->param("output_format") || 'marc'
167 unless ($commandline);
168 my $export_remove_fields = $query->param("export_remove_fields");
169 my @biblionumbers = $query->param("biblionumbers");
170 my @itemnumbers = $query->param("itemnumbers");
171 my @sql_params;
172 my $sql_query;
173 my @recordids;
175 my $StartingBiblionumber = $query->param("StartingBiblionumber");
176 my $EndingBiblionumber = $query->param("EndingBiblionumber");
177 my $itemtype = $query->param("itemtype");
178 my $start_callnumber = $query->param("start_callnumber");
179 my $end_callnumber = $query->param("end_callnumber");
180 $timestamp = ($timestamp) ? C4::Dates->new($timestamp) : ''
181 if ($commandline);
182 my $start_accession =
183 ( $query->param("start_accession") )
184 ? C4::Dates->new( $query->param("start_accession") )
185 : '';
186 my $end_accession =
187 ( $query->param("end_accession") )
188 ? C4::Dates->new( $query->param("end_accession") )
189 : '';
190 $dont_export_items = $query->param("dont_export_item")
191 unless ($commandline);
193 my $strip_nonlocal_items = $query->param("strip_nonlocal_items");
195 my $biblioitemstable =
196 ( $commandline and $deleted_barcodes )
197 ? 'deletedbiblioitems'
198 : 'biblioitems';
199 my $itemstable =
200 ( $commandline and $deleted_barcodes )
201 ? 'deleteditems'
202 : 'items';
204 my $starting_authid = $query->param('starting_authid');
205 my $ending_authid = $query->param('ending_authid');
206 my $authtype = $query->param('authtype');
207 my $filefh;
208 if ($commandline) {
209 open $filefh,"<", $id_list_file or die "cannot open $id_list_file: $!" if $id_list_file;
210 } else {
211 $filefh = $query->upload("id_list_file");
213 my %id_filter;
214 if ($filefh) {
215 while (my $number=<$filefh>){
216 $number=~s/[\r\n]*$//;
217 $id_filter{$number}=1 if $number=~/^\d+$/;
221 if ( $record_type eq 'bibs' and not @biblionumbers ) {
222 if ($timestamp) {
224 # Specific query when timestamp is used
225 # Actually it's used only with CLI and so all previous filters
226 # are not used.
227 # If one day timestamp is used via the web interface, this part will
228 # certainly have to be rewrited
229 my ( $query, $params ) = construct_query(
231 recordtype => $record_type,
232 timestamp => $timestamp,
233 biblioitemstable => $biblioitemstable,
236 $sql_query = $query;
237 @sql_params = @$params;
240 else {
241 my ( $query, $params ) = construct_query(
243 recordtype => $record_type,
244 biblioitemstable => $biblioitemstable,
245 itemstable => $itemstable,
246 StartingBiblionumber => $StartingBiblionumber,
247 EndingBiblionumber => $EndingBiblionumber,
248 branch => $branch,
249 start_callnumber => $start_callnumber,
250 end_callnumber => $end_callnumber,
251 start_accession => $start_accession,
252 end_accession => $end_accession,
253 itemtype => $itemtype,
256 $sql_query = $query;
257 @sql_params = @$params;
260 elsif ( $record_type eq 'auths' ) {
261 my ( $query, $params ) = construct_query(
263 recordtype => $record_type,
264 starting_authid => $starting_authid,
265 ending_authid => $ending_authid,
266 authtype => $authtype,
269 $sql_query = $query;
270 @sql_params = @$params;
273 elsif ( $record_type eq 'db' ) {
274 my $successful_export;
275 if ( $flags->{superlibrarian}
276 && C4::Context->config('backup_db_via_tools') )
278 $successful_export = download_backup(
280 directory => "$backupdir",
281 extension => 'sql',
282 filename => "$filename"
286 unless ($successful_export) {
287 my $remotehost = $query->remote_host();
288 $remotehost =~ s/(\n|\r)//;
289 warn
290 "A suspicious attempt was made to download the db at '$filename' by someone at "
291 . $remotehost . "\n";
293 exit;
295 elsif ( $record_type eq 'conf' ) {
296 my $successful_export;
297 if ( $flags->{superlibrarian}
298 && C4::Context->config('backup_conf_via_tools') )
300 $successful_export = download_backup(
302 directory => "$backupdir",
303 extension => 'tar',
304 filename => "$filename"
308 unless ($successful_export) {
309 my $remotehost = $query->remote_host();
310 $remotehost =~ s/(\n|\r)//;
311 warn
312 "A suspicious attempt was made to download the configuration at '$filename' by someone at "
313 . $remotehost . "\n";
315 exit;
317 elsif (@biblionumbers) {
318 push @recordids, (@biblionumbers);
320 else {
322 # Someone is trying to mess us up
323 exit;
326 unless (@biblionumbers) {
327 my $sth = $dbh->prepare($sql_query);
328 $sth->execute(@sql_params);
329 push @recordids, map {
330 map { $$_[0] } $_
331 } @{ $sth->fetchall_arrayref };
332 @recordids = grep { exists($id_filter{$_}) } @recordids if scalar(%id_filter);
335 my $xml_header_written = 0;
336 for my $recordid ( uniq @recordids ) {
337 if ($deleted_barcodes) {
338 my $q = "
339 SELECT DISTINCT barcode
340 FROM deleteditems
341 WHERE deleteditems.biblionumber = ?
343 my $sth = $dbh->prepare($q);
344 $sth->execute($recordid);
345 while ( my $row = $sth->fetchrow_array ) {
346 print "$row\n";
349 else {
350 my $record;
351 if ( $record_type eq 'bibs' ) {
352 $record = eval { GetMarcBiblio($recordid); };
354 next if $@;
355 next if not defined $record;
356 C4::Biblio::EmbedItemsInMarcBiblio( $record, $recordid,
357 \@itemnumbers )
358 unless $dont_export_items;
359 if ( $strip_nonlocal_items
360 || $limit_ind_branch
361 || $dont_export_items )
363 my ( $homebranchfield, $homebranchsubfield ) =
364 GetMarcFromKohaField( 'items.homebranch', '' );
365 for my $itemfield ( $record->field($homebranchfield) ) {
367 # if stripping nonlocal items, use loggedinuser's branch if they didn't select one
368 $branch = C4::Context->userenv->{'branch'}
369 unless $branch;
370 $record->delete_field($itemfield)
371 if ( $dont_export_items
372 || $itemfield->subfield($homebranchsubfield) ne
373 $branch );
377 elsif ( $record_type eq 'auths' ) {
378 $record = C4::AuthoritiesMarc::GetAuthority($recordid);
379 next if not defined $record;
382 if ($export_remove_fields) {
383 for my $f ( split / /, $export_remove_fields ) {
384 if ( $f =~ m/^(\d{3})(.)?$/ ) {
385 my ( $field, $subfield ) = ( $1, $2 );
387 # skip if this record doesn't have this field
388 if ( defined $record->field($field) ) {
389 if ( defined $subfield ) {
390 my @tags = $record->field($field);
391 foreach my $t (@tags) {
392 $t->delete_subfields($subfield);
395 else {
396 $record->delete_fields($field);
402 RemoveAllNsb($record) if ($clean);
403 if ( $output_format eq "xml" ) {
404 unless ($xml_header_written) {
405 MARC::File::XML->default_record_format(
407 $marcflavour eq 'UNIMARC'
408 && $record_type eq 'auths'
409 ) ? 'UNIMARCAUTH' : $marcflavour
411 print MARC::File::XML::header();
412 print "\n";
413 $xml_header_written = 1;
415 print MARC::File::XML::record($record);
416 print "\n";
418 else {
419 my (@result_build_tag) = MARC::File::USMARC::_build_tag_directory($record);
420 if ($result_build_tag[2] > 99999) {
421 warn "record (number $recordid) length ".$result_build_tag[2]." is larger than the MARC spec allows (99999 bytes)";
422 next;
424 print $record->as_usmarc();
428 if ($xml_header_written) {
429 print MARC::File::XML::footer();
430 print "\n";
433 exit;
435 elsif ( $format eq "csv" ) {
436 my @biblionumbers = uniq $query->param("biblionumbers");
437 my @itemnumbers = $query->param("itemnumbers");
438 my $output =
439 marc2csv( \@biblionumbers,
440 GetCsvProfileId( C4::Context->preference('ExportWithCsvProfile') ),
441 \@itemnumbers, );
442 print $query->header(
443 -type => 'application/octet-stream',
444 -'Content-Transfer-Encoding' => 'binary',
445 -attachment => "export.csv"
447 print $output;
448 exit;
450 } # if export
452 else {
454 my $itemtypes = GetItemTypes;
455 my @itemtypesloop;
456 foreach my $thisitemtype ( sort keys %$itemtypes ) {
457 my %row = (
458 value => $thisitemtype,
459 description => $itemtypes->{$thisitemtype}->{'description'},
461 push @itemtypesloop, \%row;
463 my $branches = GetBranches($limit_ind_branch);
464 my @branchloop;
465 for my $thisbranch (
466 sort { $branches->{$a}->{branchname} cmp $branches->{$b}->{branchname} }
467 keys %{$branches}
470 push @branchloop,
472 value => $thisbranch,
473 selected => $thisbranch eq $branch,
474 branchname => $branches->{$thisbranch}->{'branchname'},
478 my $authtypes = getauthtypes;
479 my @authtypesloop;
480 foreach my $thisauthtype ( sort keys %$authtypes ) {
481 next unless $thisauthtype;
482 my %row = (
483 value => $thisauthtype,
484 description => $authtypes->{$thisauthtype}->{'authtypetext'},
486 push @authtypesloop, \%row;
489 if ( $flags->{superlibrarian}
490 && C4::Context->config('backup_db_via_tools')
491 && $backupdir
492 && -d $backupdir )
494 $template->{VARS}->{'allow_db_export'} = 1;
495 $template->{VARS}->{'dbfiles'} = getbackupfilelist(
496 { directory => "$backupdir", extension => 'sql' } );
499 if ( $flags->{superlibrarian}
500 && C4::Context->config('backup_conf_via_tools')
501 && $backupdir
502 && -d $backupdir )
504 $template->{VARS}->{'allow_conf_export'} = 1;
505 $template->{VARS}->{'conffiles'} = getbackupfilelist(
506 { directory => "$backupdir", extension => 'tar' } );
509 $template->param(
510 branchloop => \@branchloop,
511 itemtypeloop => \@itemtypesloop,
512 authtypeloop => \@authtypesloop,
513 export_remove_fields => C4::Context->preference("ExportRemoveFields"),
516 output_html_with_http_headers $query, $cookie, $template->output;
519 sub construct_query {
520 my ($params) = @_;
522 my ( $sql_query, @sql_params );
524 if ( $params->{recordtype} eq "bibs" ) {
525 if ( $params->{timestamp} ) {
526 my $biblioitemstable = $params->{biblioitemstable};
527 $sql_query = " (
528 SELECT biblionumber
529 FROM $biblioitemstable
530 LEFT JOIN items USING(biblionumber)
531 WHERE $biblioitemstable.timestamp >= ?
532 OR items.timestamp >= ?
533 ) UNION (
534 SELECT biblionumber
535 FROM $biblioitemstable
536 LEFT JOIN deleteditems USING(biblionumber)
537 WHERE $biblioitemstable.timestamp >= ?
538 OR deleteditems.timestamp >= ?
539 ) ";
540 my $ts = $timestamp->output('iso');
541 @sql_params = ( $ts, $ts, $ts, $ts );
543 else {
544 my $biblioitemstable = $params->{biblioitemstable};
545 my $itemstable = $params->{itemstable};
546 my $StartingBiblionumber = $params->{StartingBiblionumber};
547 my $EndingBiblionumber = $params->{EndingBiblionumber};
548 my $branch = $params->{branch};
549 my $start_callnumber = $params->{start_callnumber};
550 my $end_callnumber = $params->{end_callnumber};
551 my $start_accession = $params->{start_accession};
552 my $end_accession = $params->{end_accession};
553 my $itemtype = $params->{itemtype};
554 my $items_filter =
555 $branch
556 || $start_callnumber
557 || $end_callnumber
558 || $start_accession
559 || $end_accession
560 || ( $itemtype && C4::Context->preference('item-level_itypes') );
561 $sql_query = $items_filter
562 ? "SELECT DISTINCT $biblioitemstable.biblionumber
563 FROM $biblioitemstable JOIN $itemstable
564 USING (biblionumber) WHERE 1"
565 : "SELECT $biblioitemstable.biblionumber FROM $biblioitemstable WHERE biblionumber >0 ";
567 if ($StartingBiblionumber) {
568 $sql_query .= " AND $biblioitemstable.biblionumber >= ? ";
569 push @sql_params, $StartingBiblionumber;
572 if ($EndingBiblionumber) {
573 $sql_query .= " AND $biblioitemstable.biblionumber <= ? ";
574 push @sql_params, $EndingBiblionumber;
577 if ($branch) {
578 $sql_query .= " AND homebranch = ? ";
579 push @sql_params, $branch;
582 if ($start_callnumber) {
583 $sql_query .= " AND itemcallnumber >= ? ";
584 push @sql_params, $start_callnumber;
587 if ($end_callnumber) {
588 $sql_query .= " AND itemcallnumber <= ? ";
589 push @sql_params, $end_callnumber;
591 if ($start_accession) {
592 $sql_query .= " AND dateaccessioned >= ? ";
593 push @sql_params, $start_accession->output('iso');
596 if ($end_accession) {
597 $sql_query .= " AND dateaccessioned <= ? ";
598 push @sql_params, $end_accession->output('iso');
601 if ($itemtype) {
602 $sql_query .=
603 ( C4::Context->preference('item-level_itypes') )
604 ? " AND items.itype = ? "
605 : " AND biblioitems.itemtype = ?";
606 push @sql_params, $itemtype;
610 elsif ( $params->{recordtype} eq "auths" ) {
611 if ( $params->{timestamp} ) {
613 #TODO
615 else {
616 my $starting_authid = $params->{starting_authid};
617 my $ending_authid = $params->{ending_authid};
618 my $authtype = $params->{authtype};
619 $sql_query =
620 "SELECT DISTINCT auth_header.authid FROM auth_header WHERE 1";
622 if ($starting_authid) {
623 $sql_query .= " AND auth_header.authid >= ? ";
624 push @sql_params, $starting_authid;
627 if ($ending_authid) {
628 $sql_query .= " AND auth_header.authid <= ? ";
629 push @sql_params, $ending_authid;
632 if ($authtype) {
633 $sql_query .= " AND auth_header.authtypecode = ? ";
634 push @sql_params, $authtype;
638 return ( $sql_query, \@sql_params );
641 sub getbackupfilelist {
642 my $args = shift;
643 my $directory = $args->{directory};
644 my $extension = $args->{extension};
645 my @files;
647 if ( opendir( my $dir, $directory ) ) {
648 while ( my $file = readdir($dir) ) {
649 next unless ( $file =~ m/\.$extension(\.(gz|bz2|xz))?/ );
650 push @files, $file
651 if ( -f "$directory/$file" && -r "$directory/$file" );
653 closedir($dir);
655 return \@files;
658 sub download_backup {
659 my $args = shift;
660 my $directory = $args->{directory};
661 my $extension = $args->{extension};
662 my $filename = $args->{filename};
664 return unless ( $directory && -d $directory );
665 return unless ( $filename =~ m/\.$extension(\.(gz|bz2|xz))?$/ );
666 return if ( $filename =~ m#/# );
667 $filename = "$directory/$filename";
668 return unless ( -f $filename && -r $filename );
669 return unless ( open( my $dump, '<', $filename ) );
670 binmode $dump;
672 while ( read( $dump, my $data, 64 * 1024 ) ) {
673 print $data;
675 close($dump);
676 return 1;