package Mkcd::Build; my $VERSION = '3.0.0'; use strict; use File::NCopy qw(copy); use File::Path; use URPM qw(ranges_overlap); use Mkcd::Package qw(rpmVersionCompare); use Mkcd::Tools qw(log_ find_list); use Mkcd::Optimize qw(optimize_space get_pkgs_deps); use MDK::Common qw(any if_); my $MIN_CHUNK = 0.0001; =head1 NAME Build - mkcd module =head1 SYNOPSYS require Mkcd::Build; =head1 DESCRIPTION C include the mkcd packages list functions. =head1 SEE ALSO mkcd =head1 COPYRIGHT Copyright (C) 2000-2004 Mandrakesoft This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. =cut my $config; sub new { my ($class, $conf) = @_; $config = $conf; bless { config => $config, }, $class; } sub addRPMToDiff { my ($rpm, $rpmd, $diff, $cdnum, $repnumber, $i, $list, $curdir, $size, $rpmsize, $totrpmsize, $repnum, $done, $reallist, $realrepnum, $group) = @_; my @data; for (my $s; $s < @$rpm; $s++) { push @data, [$rpm->[$s],1, $rpmd->[$s], $rpmsize->[$s]]; log_("addRPMToDiff: $rpm->[$s] put in rep $repnumber ($done->{rep})\n", $config->{verbose}, $config->{LOG}, 4); $done->{rep}{$rpm->[$s]} = $repnumber; $done->{list}{$rpm->[$s]} = $list; } my $diff_data = [ $curdir, $i, $list, $repnum, 1, \@data, $rpmd, $totrpmsize, $reallist ]; my $idx = push @{$diff->{data}}, $diff_data; push @{$diff->{idx}}, --$idx; push @{$group->{reploc}{$realrepnum}{diff}{$repnum}}, $diff_data if $realrepnum != $repnum; $size->{disc}[$cdnum] += $totrpmsize; $size->{rep}{$cdnum}{$curdir->[1]}{$list} += $totrpmsize; log_("addRPMToDiff: SIZE disc $cdnum: $size->{disc}[$cdnum] (+ @$rpm $totrpmsize ID $idx) added rpmd $rpmd\n", $config->{verbose}, $config->{LOG}, 3); 1 } sub check_last_relocatable { my ($group, $cd, $repname, $repnum, $list, $curdir_old, $old) = @_; my ($curdir, $new_repnum, $newlist); if (defined $group->{reploc}{$repnum}{last}{list} && $group->{reploc}{$repnum}{last}{list} == $list) { $new_repnum = $group->{reploc}{$repnum}{last}{rep}; log_("check_last_relocatable: using previous relocatable dir $new_repnum\n",1 , $config->{LOG}, 1); $curdir = $group->{reploc}{$repnum}{last}{curdir}; $newlist = $group->{reploc}{$repnum}{last}{newlist} } else { my $last = $group->{reploc}{$repnum}{last}{rep} || $repnum - 1; $new_repnum = $last+$MIN_CHUNK; log_("check_last_relocatable: using new relocatable dir $new_repnum cd $cd repname $repname repnum $repnum\n",1 , $config->{LOG}, 1); $group->{reploc}{$repnum}{last}{rep} = $new_repnum; $group->{reploc}{$repnum}{last}{list} = $list; $newlist = @{$config->{list}}; $config->{list}[$newlist] = {}; push @{$group->{reploc}{$repnum}{list}}, [ $new_repnum, $cd, $repname, $newlist ]; $curdir = [ $cd, $repname ]; push @$curdir, $curdir_old->[2], $curdir_old->[3] if ref $curdir_old; $group->{reploc}{$repnum}{last}{newlist} = $newlist; $group->{reploc}{$repnum}{last}{curdir} = $curdir; $group->{reploc}{$repnum}{curdir}{$new_repnum} = $curdir; $group->{reploc}{$repnum}{newlist}{$new_repnum} = $newlist; $group->{reploc}{$repnum}{reallist}{$new_repnum} = $list; $group->{reploc}{$repnum}{old}{$new_repnum} = $old } return $new_repnum, $curdir, $newlist } sub add_relocatable_package { my ($group, $tcd, $diff, $r, $list, $i, $repnum, $repname) = @_; my $done = $group->{done}; log_("add_relocatable_package: going in relocatable mode for deps $r before $repnum\n",1 , $config->{LOG}, 1); my ($new_repnum, $curdir, $newlist) = check_last_relocatable($group, @{$group->{reverse_rep}{rpm}{$tcd}}{'cd', 'name'}, $repnum, $list, 0, 1); log_("add_relocatable_package: $r put in rep $new_repnum list $list\n", $config->{verbose}, $config->{LOG}, 4); $done->{rep}{$r} = $new_repnum; $done->{list}{$r} = $newlist; my $totrpmsize = $group->{size}{$r}{$list}[0]; $group->{reploc}{$repnum}{size}{$new_repnum} += $totrpmsize; my $rpmd = [[ $r, 0, 0 ]]; my $data = [[ $r, 1, 0, 0 ]]; my $diff_data = [ $curdir, $i, $newlist, $new_repnum, 1, $data, $rpmd, $totrpmsize, $list ]; my $idx = push @{$diff->{data}}, $diff_data; push @{$diff->{idx}}, --$idx; push @{$group->{reploc}{$repnum}{diff}{$new_repnum}}, $diff_data; $new_repnum } sub all_rejected { my ($all_l, $r) = @_; foreach my $l (@$all_l) { return 0 if !$r->{$l} } 1 } sub processDeps { my ($r, $group, $cdnum, $repname, $done, $rpmlist, $topush, $intopush, $depsdisc, $rpmd, $list, $loop, $i, $tobedone, $buildlist, $rpm, $needed, $diff) = @_; # FIXME I think that this is wrong, but not really sure # It may happen that package get rejected because the non-done deps could not, # for any reason, be used, and this package is not put after the done deps. # However I guess that the put_in_rep loop over directories may save the # stuff. # FIXME This is not correct, should iterate on the lists to find if an entry exist (not important) #if (! defined $done->{rep}{$r} && ! defined $rpmlist->[$i]{$done->{list}{$r}}{$r}) { # log_("WARNING THIS SHOULD NOT HAPPEN processDeps: adding a non programmed deps $r\n", $config->{verbose}, $config->{LOG}, 1); #} my $repnum = $group->{orderedrep}{rpm}{"$cdnum/$repname"}; my $tcd = $done->{rep}{$r} if !$rpmlist->[$i]{$done->{list}{$r}}{$r}{noprovide} || $done->{rep}{$r} <= $repnum; log_("processDeps: deps $r (tcd $tcd done $done->{rep}{$r})\n", $config->{verbose}, $config->{LOG}, 3); if ($tcd > $repnum && $rpmlist->[$i]{$done->{list}{$r}}{$r}{relocatable}) { $tcd = add_relocatable_package($group, $tcd, $diff, $r, $done->{list}{$r}, $i, $group->{orderedrep}{rpm}{"$cdnum/$repname"}, $repname) } my ($r_l, $all_l) = find_list($config, $group, $r, $list, !$tcd); if (all_rejected($all_l, $group->{rejected}{$r})) { log_("ERROR processDeps: deps $r rejected, rejecting @$rpm\n",1 , $config->{LOG}, 1); log_("Rejecting @$rpm $r\n", $config->{verbose}, $config->{LOG},1); foreach (@$rpm) { foreach my $t_l (@$all_l) { push @{$group->{rejected}{$_}{$t_l}}, [ 'deps_rejected', $r] } } $$loop = 1; %$topush = (); return 0 } if ($tcd) { if ($tcd > $$depsdisc) { $$depsdisc = $tcd }; log_("processDeps: deps done $r on rep $tcd ($$depsdisc) list $done->{list}{$r}\n", $config->{verbose}, $config->{LOG}, 4); return 2 } foreach my $l (@$all_l) { if ($tobedone->{$r}{$l}) { if ($l == $list) { log_("$r tobedone\n", $config->{verbose}, $config->{LOG},3); if ($intopush->{$r}) { log_("WARNING processDeps: $r added twice\n", $config->{verbose}, $config->{LOG}, 3); return 1 } push @$rpmd, [$r, $rpmlist->[$i]{$l}{$r}]; $intopush->{$r} = 1; push @{$topush->{$l}}, $rpmd; log_("processDeps: adding looping deps $r ($l) with @$rpm\n", $config->{verbose}, $config->{LOG},3); return } else { if ($group->{listmatrix}{rpm}{$list}{$l} && !$group->{rejected}{$r}{$l}) { # FIXME tobedone may not mean dependencies loop in parallel mode for different list. log_("processDeps: $r is already scheduled on list $l, waiting.\n", $config->{verbose}, $config->{LOG},4); %$topush = (); push @{$buildlist->[$i]{$list}}, @$rpmd > 1 ? $rpmd : $rpmd->[0]; return 3 #$intopush{$r} and log_("ERROR: $r added twice\n",1, $config->{LOG}) and return 0; #$intopush{$r} = 1; #push @{$topush{$l}}, [$r, $rpmlist->[$i]{$l}{$r}]; #log_("DEPS $r ($_ -- $l)\n", $config->{verbose}, $config->{LOG}) } else { log_("ERROR processDeps: deps $r could not be put in directory before packages @$rpm\n", $config->{verbose}, $config->{LOG},1); reject_rpm($rpm, $l, $group, 'order_pb', $r, \$loop, $topush); next } } } else { if ($l == $list) { if ($intopush->{$r}) { log_("WARNING processDeps: $r added twice\n", 1, $config->{LOG}, 2); return 1 } $intopush->{$r} = 1; push @{$topush->{$l}}, [$r, $rpmlist->[$i]{$l}{$r}]; log_("processDeps: adding normal deps $r ($_ -- $l)\n", $config->{verbose}, $config->{LOG},3); return } else { if ($config->{list}[$l]{done}) { log_("processDeps: list $l of deps $r is done, doing nothing.\n", $config->{verbose}, $config->{LOG},6); return } if ($group->{options}{sequential}) { log_("WARNING processDeps: could not add interlist deps in sequential mode\n",1, $config->{LOG},2); reject_rpm($rpm, $l, $group, 'sequential', $r, \$loop, $topush); next } else { if (defined $needed->{$list}{asap} && grep { $l == $_->[0] } @{$needed->{$list}{asap}}) { log_("ERROR processDeps: list of deps $r is already waiting for @$rpm list\n",1, $config->{LOG},1); reject_rpm($rpm, $l, $group, 'order_pb', $r, \$loop); next } else { if ($group->{listmatrix}{rpm}{$list}{$l}) { if ($intopush->{$r}) { log_("WARNING processDeps: $r added twice\n",1, $config->{LOG},2); return 1 } $intopush->{$r} = 1; push @{$topush->{$l}}, [$r, $rpmlist->[$i]{$l}{$r}]; log_("processDeps: adding normal deps $r ($_ -- $l)\n", $config->{verbose}, $config->{LOG},3); return } else { log_("ERROR processDeps: deps $r could not be put in directory before packages @$rpm\n",1, $config->{LOG},1); reject_rpm($rpm, $l, $group, 'order_pb', $r, \$loop); next } } } } } } } sub reject_rpm { my ($rpm, $list, $group, $reason, $r, $loop, $topush) = @_; log_("Rejecting @$rpm\n", $config->{verbose}, $config->{LOG},1); foreach (@$rpm) { push @{$group->{rejected}{$_}{$list}}, [ $reason, $r ] } %$topush = (); $$loop = 1; } sub updateGenericLimit { my ($groups, $cdsize) = @_; log_("updateGenericLimit\n", $config->{verbose}, $config->{LOG},2); for (my $i; $i < @$groups; $i++) { foreach my $type (keys %{$groups->[$i]{orderedlist}}) { foreach my $list (@{$groups->[$i]{orderedlist}{$type}}) { foreach my $r (@{$groups->[$i]{list}{$list}{$type}}) { my ($cd, $rep, $repopt) = @$r; #log_("trying to update disc $cd rep $rep list $list limit repopt $repopt (",1, $config->{LOG}),keys %$repopt,") opt $opt (",keys %$opt,")\n"; $config->{list}[$list]{disc}{$cd}{$rep}{done} and next; $repopt->{limit} or next; $repopt->{limit}{size} = $repopt->{limit}{value} * $cdsize->[$cd]; log_("updateGenericLimit: setting disc $cd rep $rep list $list limit to $repopt->{limit}{size} ($repopt->{limit}{value} * $cdsize->[$cd])\n", $config->{verbose}, $config->{LOG}, 3); } } } } } sub testSoftLimit { my ($opt, $cd, $groups, $buildlist) = @_; log_("testSoftLimit\n", $config->{verbose}, $config->{LOG}, 2); my $softnok = 1; # FIXME this code must be tested if ($opt->{limit} && $opt->{limit}{soft}) { foreach my $l (@{$config->{disc}[$cd]{fastgeneric}}) { my $lst = $l->[2]{list}; for (my $i; $i < @$groups; $i++) { $groups->[$i]{list}{$lst} or next; $softnok = 0 if (@{$buildlist->[$i]{$lst}} && !($lst->{limit} && $lst->{limit}{soft})) } } } return $softnok; } sub add_one_disc { my ($cdlists, $group, $cdsize, $list, $cds, $sources, $size, $g) = @_; my $ncd; foreach (keys %$cdlists) { $ncd = $_ + 1 if $ncd <= $_ } log_("add_one_disc: $config->{list}[$list]{cd} -- $ncd\n", $config->{verbose}, $config->{LOG}, 3); if (!$config->{list}[$list]{cd} || $config->{list}[$list]{cd} >= $ncd) { log_("add_one_disc: adding new disc $ncd\n", $config->{verbose}, $config->{LOG}, 4); $config->{disc}[$ncd]{size} = $config->{discsize}; my $functions = $config->{group}{disc}{functions}{functions}; $cdsize->[$ncd] = $config->{discsize}; $config->{disc}[$ncd]{name} = $ncd; $size->{optimize_space}{disc}{$ncd} = $cdsize->[$ncd]; $group->{disc_impacted}{$ncd} = 1; my ($curdir, $srpmcurdir); my $tmp = "$config->{tmp}/build/$config->{name}"; my $f = "$tmp/$ncd.list"; -f $f and unlink $f; if ($config->{nolive}) { log_("makeDisc: removing $tmp/$ncd\n", $config->{verbose}, $config->{LOG}, 3); rmtree "$tmp/$ncd"; mkpath "$tmp/$ncd"; } else { my $dir = "$config->{topdir}/build/$config->{name}"; rmtree "$dir/$ncd"; rmtree "$dir/first/$ncd"; mkpath "$dir/$ncd" } my $instcd = $group->{installDisc}; my ($rep, $src_rep); my $struct_v = $config->{struct_version}; my $media_srpm = $config->{struct}{$struct_v}{srpm_media}; my $media_dir = $config->{struct}{$struct_v}{media}; if ($sources && $config->{list}[$list]{sources} && $config->{list}[$list]{sources}{separate}) { $config->{disc}[$ncd]{serial} = "$config->{name}-$ncd-src"; $config->{disc}[$ncd]{longname} = "Mandrakelinux $config->{name} sources"; $config->{disc}[$ncd]{appname} = "Mandrakelinux $config->{name} sources disc $ncd"; $config->{disc}[$ncd]{label} = substr "$config->{name}-src-Disc$ncd", 0, 32; $config->{disc}[$ncd]{group_list}{$g}{$list}{srpm} = 1; &{$functions->{dir}[0][5]}($ncd, 3, "srpms", $media_srpm); &{$functions->{generic}[0][5]}($ncd, 4, "srpms",1); &{$functions->{generic}[1][5]}($ncd, 5, { source => 1 }); push @{$config->{disc}[$instcd]{function}{data}{installation}[1]{srpmsdir}}, [ 0, $ncd, "srpms" ]; $srpmcurdir = [ $ncd, "srpms" ]; push @{$group->{list}{$list}{srpm}}, $srpmcurdir; $src_rep = $group->{maxrep}{srpm}; push @{$group->{replist}{srpm}}, [ $ncd, 'srpms', $group->{maxrep}{srpm}++, { $list => $srpmcurdir } ]; } else { $config->{disc}[$ncd]{serial} = "$config->{name}-$ncd"; $config->{disc}[$ncd]{longname} = "Mandrakelinux $config->{name}"; $config->{disc}[$ncd]{appname} = "Mandrakelinux $config->{name} disc $ncd"; $config->{disc}[$ncd]{label} = substr "$config->{name}-Disc$ncd", 0, 32; $config->{disc}[$ncd]{group_list}{$g}{$list}{rpm} = 1; &{$functions->{dir}[0][5]}($ncd, 1, "rpms","$media_dir$ncd"); &{$functions->{generic}[0][5]}($ncd, 2, "rpms", 1); $group->{orderedrep}{rpm}{"$ncd/rpms"} = $ncd; # # generic has no FIXED part, otherwize a call to generic with fixed=0 # would have been needed # $curdir = [$ncd, "rpms"]; push @{$group->{list}{$list}{rpm}}, $curdir; $rep = $group->{maxrep}{rpm}; if ($group->{replist}{rpm}[$group->{maxrep}{rpm}-1]) { die "FATAL add_one_disc: rep $group->{maxrep}{rpm} should not exist !\n" } else { $group->{replist}{rpm}[$group->{maxrep}{rpm}-1], [ $ncd, 'rpms', $group->{maxrep}{rpm}++, { $list => $curdir } ]; } push @{$config->{disc}[$instcd]{function}{data}{installation}[1]{rpmsdir}}, [ 0, $ncd, "rpms" ]; if ($config->{list}[$list]{sources}) { &{$functions->{dir}[0][5]}($ncd,3, "srpms", $media_srpm); &{$functions->{generic}[0][5]}($ncd,4, "srpms",1); &{$functions->{generic}[1][5]}($ncd,5, { source => 1 }); push @{$config->{disc}[$instcd]{function}{data}{installation}[1]{srpmsdir}}, [ 0, $ncd, "srpms" ]; $srpmcurdir = [ $ncd, "srpms" ]; $src_rep = $group->{maxrep}{srpm}; if ($group->{replist}{srpm}[$group->{maxrep}{srpm}-1]) { die "FATAL add_one_disc: rep $group->{maxrep}{srpm} should not exist !\n" } else { $group->{replist}{srpm}[$group->{maxrep}{srpm}-1], [ $ncd, 'srpms', $group->{maxrep}{srpm}++, { $list => $srpmcurdir } ]; } push @{$group->{list}{$list}{srpm}}, $srpmcurdir } } push @$cds, $ncd; $cdlists->{$ncd} = 2; return $curdir, $rep, $srpmcurdir, $src_rep } else { return 0 } } sub addSRPMToDiff { my ($rpmd, $done, $diff, $size, $srpmrep, $srpmsize, $curdir, $srpm, $list, $i, $cdnum) = @_; for (my $s; $s < @$rpmd; $s++) { if (!$rpmd->[$s][1]{nosrc} && !$done->{rep}{$srpm->[$s]}) { my $srep = $srpmrep->{$srpm->[$s]}; my $idx = push @{$diff->{data}}, [ $srep->[2], $i, $list, $srep->[1], 2, [[$srpm->[$s],1, $rpmd->[$s], $srpmsize->[$s]]], 0, $srpmsize->[$s] ]; push @{$diff->{idx}}, $idx - 1; $size->{disc}[$srep->[0]] += $srpmsize->[$s]; $size->{rep}{$srep->[0]}{$srep->[2][1]}{$list} += $srpmsize->[$s]; log_("SIZE disc $srep->[0]: $size->{disc}[$srep->[0]] (+ $srpm->[$s] $srpmsize->[$s])\n", $config->{verbose}, $config->{LOG}, 2); } $done->{rep}{$srpm->[$s]}++; $done->{list}{$srpm->[$s]} = $list } 1 } sub sourcesSizeCheck { my ($done, $rpmd, $srpm, $group, $groups, $size, $cdsize, $list, $cdlists, $cdnum, $rpmsize, $buildlist, $cds, $i, $diff) = @_; my %srpmrep; my $srpmok = 1; my @srpmsize; for (my $s; $s < @$srpm; $s++) { $done->{rep}{$srpm->[$s]} and next; $rpmd->[$s][1]{nosrc} and next; my $srpmsize = $group->{size}{$srpm->[$s]}{$list}[0]; $srpmsize[$s] = $srpmsize; for (my $k; $k < @{$group->{list}{$list}{srpm}}; $k++) { my $srpmdir = $group->{list}{$list}{srpm}[$k]; my ($srccd, $srcrepname, $srcopt) = @$srpmdir; my $src_rep_num = $group->{orderedrep}{srpm}{"$srccd/$srcrepname"}; log_("trying source disc $srccd\n", $config->{verbose}, $config->{LOG}, 2); $cdlists->{$srccd} > 1 or next; my $currentrpm; $cdnum == $srccd and $currentrpm = $rpmsize; my $softnok = testSoftLimit($srcopt, $srccd, $groups, $buildlist); my $gain = $size->{disc}[$srccd] + $srpmsize + $currentrpm - $cdsize->[$srccd]; # FIXME this need to be tested if ($gain <= 0 && !($srcopt->{limit} && ($softnok || !$srcopt->{limit}{soft}) && $size->{rep}{$srccd}{$srcrepname}{$list} > $srcopt->{limit}{size})) { $srpmrep{$srpm->[$s]} = [$srccd, $src_rep_num, $srpmdir]; last } elsif ($k == $#{$group->{list}{$list}{srpm}}) { if (optimize_space($config, $groups, $diff, $size, $cdsize, $srccd, $gain, $cdlists,0, $i, $list, 'srpm', $srpmsize + $currentrpm) < $gain) { $srpmok = 0 } else { $done = $groups->[$i]{done}; $srpmrep{$srpm->[$s]} = [$srccd, $src_rep_num, $srpmdir]; } } } if (!$srpmrep{$srpm->[$s]}) { $srpmok = 0 # no last here because if in autoMode a CD will be added after and we will not retest for each srpm if it could be put on an existing CD. } } if (!$srpmok && $config->{list}[$list]{auto}) { my (undef, undef, $srpmdir, $repnum) = add_one_disc($cdlists, $group, $cdsize, $list, $cds, 1, $size, $i); if ($srpmdir) { for (my $s; $s < @$srpm; $s++) { if (!$srpmrep{$srpm->[$s]}) { $srpmrep{$srpm->[$s]} = [$srpmdir->[0], $repnum, $srpmdir]; } } $srpmok = 1 } } return \%srpmrep, \@srpmsize, $srpmok } sub choose_alt { my ($deps, $rpmlist, $group, $cdnum, $repname, $list, $buildlist, $intopush, $tobedone, $needed, $size_matter) = @_; my $r = -1; my $score = [ 0, $group->{maxlist} ]; my $done = $group->{done}; my $all_rejected = 1; DEPS: { foreach my $testalt (1,0) { foreach (@$deps) { # FIXME it may have a problem here, as depslistid are not erased when the # package is removed, that is to say that if the previous deps failed for # any reason, alternates deps may be added, although excluded before # however this _must_ not happen, and signify a bug somewhere else. # Update: This may happen when the schedule deps has been rejected, and thus # only the previous not selected one coud fulfil the require. my $pkg = $group->{depslistid}[$_]; my (undef, $all_l) = find_list($config, $group, $pkg, $list); foreach my $pkg_list (@$all_l) { if ($group->{rejected}{$pkg}{$pkg_list}) { log_("choose_alt: $pkg is rejected, ignoring\n", $config->{verbose}, $config->{LOG}, 6); last } $all_rejected = 0; if ($testalt && ! defined $rpmlist->{$pkg_list}{$pkg} || $rpmlist->{$pkg_list}{$pkg}{noalternatives}) { log_("choose_alt: $pkg is not selected in first pass for alternatives\n", $config->{verbose}, $config->{LOG},6); last } $intopush->{$pkg} and $r = $pkg and last DEPS; log_("choose_alt: alternatives deps $pkg (noprovide $rpmlist->{$done->{list}{$pkg}}{$pkg}{noprovide} done $done->{rep}{$pkg} orderedrep " . $group->{orderedrep}{rpm}{"$cdnum/$repname"} . ")\n", $config->{verbose}, $config->{LOG}, 6); my $tcd; if ($rpmlist->{$done->{list}{$pkg}}{$pkg}{relocatable}) { $tcd = $group->{orderedrep}{rpm}{"$cdnum/$repname"} } elsif (!$rpmlist->{$done->{list}{$pkg}}{$pkg}{noprovide} || $done->{rep}{$pkg} <= $group->{orderedrep}{rpm}{"$cdnum/$repname"}) { $tcd = $done->{rep}{$pkg} } if ($tcd && $tcd <= $group->{orderedrep}{rpm}{"$cdnum/$repname"}) { log_("$pkg ($tcd) done\n", $config->{verbose}, $config->{LOG}, 6); $r = 0; last DEPS } my $s = $size_matter ? $group->{size}{$pkg}{$pkg_list}[0] : $group->{scorelist}{$pkg}; #log_("choose_alt: pkg $pkg buildlist $buildlist list $pkgList other list $list tcd $tcd list $l ($buildlist->{$pkgList})\n", $config->{verbose}, $config->{LOG}, 6); #log_("choose_alt: ($needed->{$pkgList}{asap})\n", $config->{verbose}, $config->{LOG}, 6); if ($pkg_list && $list != $pkg_list && (defined $needed->{$pkg_list}{asap} && (any { if (ref $_) { $list == $_->[0] } else { log_("ERROR choose_alt: [$_] should be a reference (pkg $pkg list $pkg_list asap [@{$needed->{$pkg_list}{asap}}])\n", $config->{verbose}, $config->{LOG}, 6) } } @{$needed->{$pkg_list}{asap}}) || ($group->{options}{sequential} && !$config->{list}[$pkg_list]{done} && @{$buildlist->{$pkg_list}})) ) { next } log_("choose_alt: $pkg list $pkg_list (tcd $tcd listmatrix $group->{listmatrix}{rpm}{$list}{$pkg_list} listsort $group->{listsort}{$pkg_list}{rpm} score->[1] $score->[1] s $s)\n", $config->{verbose}, $config->{LOG}, 6); if (!$tcd && $group->{listmatrix}{rpm}{$list}{$pkg_list}) { if ($group->{listsort}{$pkg_list}{rpm} < $score->[1] || $group->{listsort}{$pkg_list}{rpm} == $score->[1] && $s > $score->[0]) { log_("choose_alt: choosing $pkg ($s, $group->{listsort}{$pkg_list}{rpm})\n", $config->{verbose}, $config->{LOG}, 6); $score = [ $s, $group->{listsort}{$pkg_list}{rpm} ]; $r = $pkg; last DEPS if $tobedone->{$r}{$pkg_list}; last } } } } last if $r != -1 && $r } } return $r, $all_rejected } sub check_deps { my ($rpmd, $group, $done, $rpmlist, $list, $i, $tobedone, $buildlist, $rpm, $cdnum, $repname, $needed, $rep_num, $diff) = @_; log_("check_deps\n", $config->{verbose}, $config->{LOG}, 5); my $deps = get_pkgs_deps($rpmd, $group); my $loop; if (@$deps) { my ($waiting, %topush, %intopush, $depsdisc); foreach (@$deps) { if (!ref $_) { my $a = processDeps($group->{depslistid}[$_], $group, $cdnum, $repname, $done, $rpmlist, \%topush, \%intopush, \$depsdisc, $rpmd, $list, \$loop, $i, $tobedone, $buildlist, $rpm, $needed->[$i], $diff); if ($a < 0) { return 0 } elsif ($a == 0) { last } elsif ($a == 2) { next } elsif ($a == 3) { $waiting = 1; last } } else { # must create a virtual package that install all of them in one loop log_("check_deps: alternatives deps @$_\n", $config->{verbose}, $config->{LOG}, 5); my ($r, $all_rejected) = choose_alt($_, $rpmlist->[$i], $group, $cdnum, $repname, $list, $buildlist->[$i], \%intopush, $tobedone, $needed->[$i]); $intopush{$r} and next; if ($r == -1) { my $reject = 0; my $msg; if ($all_rejected) { $msg = "all alternatives deps (@$_) rejected"; $reject = 'deps_rejected' } elsif ($group->{orderedrep}{rpm}{"$cdnum/$repname"} >= $group->{listmaxrep}{rpm}{$list}) { $msg = "ERROR check_deps: alternatives deps (@$_) could not be put in directory before packages @$rpm\n"; $reject = 'order_pb' } else { $loop = 2 } if ($reject) { my $deps_rpm = join ' ',map { $group->{depslistid}[$_] } @$_; log_("Rejecting @$rpm\n", $config->{verbose}, $config->{LOG},2); foreach my $p (@$rpm) { push @{$group->{rejected}{$p}{$list}}, [ $reject, $deps_rpm ] } %topush = (); $loop = 1; } last } if ($r) { my $a = processDeps($r, $group, $cdnum, $repname, $done, $rpmlist, \%topush, \%intopush, \$depsdisc, $rpmd, $list, \$loop, $i, $tobedone, $buildlist, $rpm, $needed->[$i]); if ($a < 0) { return 0 } elsif ($a == 0) { last } elsif ($a == 2) { next } elsif ($a == 3) { $waiting = 1; last } } else { log_("Finding better alternatives rep (@$_ - $depsdisc)\n", $config->{verbose}, $config->{LOG}, 4); my $bestdisc = keys %{$group->{orderedrep}{rpm}}; if ($bestdisc >= $depsdisc) { foreach (@$_) { my $pkg = $group->{depslistid}[$_]; my ($r_l, $all_l) = find_list($config, $group, $pkg, $list); if (all_rejected($all_l, $group->{rejected}{$pkg})) { log_("$pkg rejected\n", $config->{verbose}, $config->{LOG}, 2); next } my $tcd; log_("$pkg done $done->{list}{$pkg} relocatable $rpmlist->[$i]{$done->{list}{$pkg}}{$pkg}{relocatable}\n", $config->{verbose}, $config->{LOG}, 4); if ($rpmlist->[$i]{$done->{list}{$pkg}}{$pkg}{relocatable}) { $tcd = $rep_num } else { $tcd = $done->{rep}{$pkg}; } $tcd or next; log_("$pkg => rep $tcd\n", $config->{verbose}, $config->{LOG}, 4); if ($tcd < $bestdisc) { $bestdisc = $tcd } } $bestdisc > $depsdisc and $depsdisc = $bestdisc } log_("Finding better alternatives rep result $depsdisc\n", $config->{verbose}, $config->{LOG}, 4); } } } $waiting and return 1; if (keys %topush) { $loop = 1; log_("Adding dependencies, looping\n", $config->{verbose}, $config->{LOG}, 3); my $test = @$rpmd > 1 ? $rpmd : $rpmd->[0]; push @{$buildlist->[$i]{$list}}, (@$rpmd > 1 ? $rpmd : $rpmd->[0]); foreach (keys %topush) { $list != $_ and push @{$needed->[$i]{$list}{asap}}, [ $_, int @{$buildlist->[$i]{$_}} ]; push @{$buildlist->[$i]{$_}}, @{$topush{$_}} } } elsif ($rep_num < $depsdisc) { if ($group->{listmaxrep}{rpm}{$list} >= $depsdisc) { $loop = 2; # has a chance to put it after depsdisc log_("Dependencies on further directories ($depsdisc < $group->{listmaxrep}{rpm}{$list} rep_num $rep_num)\n", $config->{verbose}, $config->{LOG}, 3); } else { $loop = 1; log_("check_deps: dependencies are in further directories, rejecting @$rpm\n", $config->{verbose}, $config->{LOG}, 2); foreach (@$rpm) { push @{$group->{rejected}{$_}{$list}}, ["order_pb", ""] } } } } $loop } sub put_in_rep { my ($i, $groups, $group, $size, $rpmsize, $all_rpmsize, $cdsize, $needed, $rpm, $rpmd, $list, $cdlists, $buildlist, $diff, $cds, $done, $tobedone, $rpmlist, $nosrcfit) = @_; my $loop; my $dn; my $reject_reason; log_("put_in_rep: @$rpm\n", $config->{verbose}, $config->{LOG}, 3); for (my $j; !$loop && !$dn && $j < @{$group->{list}{$list}{rpm}}; $j++) { $loop = 0; log_("put_in_rep: testing dir $j\n", $config->{verbose}, $config->{LOG}, 3); my $curdir = $group->{list}{$list}{rpm}[$j]; $config->{list}[$list]{disc}{$curdir->[0]}{$curdir->[1]}{done} and next; my ($cdnum, $repname, $repopt) = @$curdir; my $rep_num = $group->{orderedrep}{rpm}{"$cdnum/$repname"}; log_("put_in_rep: testing dir $j cdnum $cdnum repname $repname\n", $config->{verbose}, $config->{LOG}, 3); $cdlists->{$cdnum} > 1 or next; my $not_good_rep; foreach my $r (@$rpmd) { if (defined $r->[1]{notinrep} && $r->[1]{notintrep} == $rep_num || defined $r->[1]{inrep} && $r->[1]{inrep} != $rep_num) { $not_good_rep = 1; next } } if ($not_good_rep) { if ($rep_num == $group->{listmaxrep}{rpm}{$list}) { log_("ERROR put_in_rep: could not handle inrep or notinrep flags for rpms @$rpm", $config->{verbose}, $config->{LOG}, 1); $reject_reason = 'config' } next } my ($relocatable_list, $relocatable_rep_num) = ($list, $rep_num); if (defined $group->{reploc}{$rep_num} && ref $group->{reploc}{$rep_num}) { ($relocatable_rep_num, $curdir, $relocatable_list) = check_last_relocatable($group, $cdnum, $repname, $rep_num, $list, $curdir); } my $softnok = testSoftLimit($repopt, $cdnum, $groups, $buildlist); my $gain = $size->{disc}[$cdnum] + $all_rpmsize - $cdsize->[$cdnum]; log_("put_in_rep: curdir cd $cdnum (space $gain rpm size $all_rpmsize) rep $repname rep_num $rep_num softnok $softnok\n", $config->{verbose}, $config->{LOG}, 4); if ($gain > 0 || $repopt->{limit} && ($softnok || !$repopt->{limit}{soft}) && $size->{rep}{$cdnum}{$repname}{$list} + $all_rpmsize > $repopt->{limit}{size}) { if ($j == $#{$group->{list}{$list}{rpm}}) { if (!($repopt->{limit} && !$softnok && $repopt->{limit}{soft})) { if (optimize_space($config, $groups, $diff, $size, $cdsize, $cdnum, $gain, $cdlists,0, $i, $list, 'rpm', $all_rpmsize) < $gain) { # done is rebuilt inside optimize_space and the referece needs to be locally updated if ($config->{list}[$list]{auto}) { ($curdir, $rep_num) = add_one_disc($cdlists, $group, $cdsize, $list, $cds,0, $size, $i); if ($curdir) { $cdnum = $curdir->[0] } else { log_("Could not add more disc, rejecting @$rpm\n", $config->{verbose}, $config->{LOG}, 2); foreach my $p (@$rpm) { push @{$group->{rejected}{$p}{$list}}, [ "no_disc", "" ] } next } } else { log_("Rejecting @$rpm\n", $config->{verbose}, $config->{LOG}, 2); foreach my $p (@$rpm) { push @{$group->{rejected}{$p}{$list}}, ["no_space", ""] } next } } else { $done = $groups->[$i]{done} } } else { foreach my $l (@{$config->{disc}[$cdnum]{fastgeneric}}) { my $lst = $l->[2]{list}; $list == $lst and next; for (my $grp; $grp < @$groups; $grp++) { $groups->[$grp]{list}{$lst}{rpm} or next; push(@{$needed->[$grp]{$list}{asap}}, [ $lst, 0 ]) if (!($lst->{limit} && $lst->{limit}{soft})) } } } } else { next } } if (!$config->{nodeps} && !$group->{options}{nodeps}) { $loop = check_deps($rpmd, $group, $done, $rpmlist, $list, $i, $tobedone->[$i], $buildlist, $rpm, $cdnum, $repname, $needed, $rep_num, $diff) } if ($loop) { $loop = 0 if $loop == 2; next } log_("@$rpm deps ok\n", $config->{verbose}, $config->{LOG}, 4); my $nosrc = 1; my @srpm; my $donesrpm = 1; if (!$group->{options}{nosources} && @{$group->{list}{$list}{srpm}}) { for (my $s; $s < @$rpmd; $s++) { my $srpm = $group->{urpm}{sourcerpm}{$rpm->[$s]}; $srpm =~ s/\.rpm$//; if (!$group->{size}{$srpm}{$list}) { log_("put_in_rep ERROR: $srpm not available, trying alternatives => ", $config->{verbose}, $config->{LOG}, 5); my ($srpmname) = $srpm =~ /(.*)-[^-]+-[^-]+\.src/; $srpm = $group->{srpmname}{$srpmname}; if ($group->{size}{$srpm}{$list}) { log_(" $srpm\n", $config->{verbose}, $config->{LOG}) } else { if ($srpm) { log_("not found (but a srpm $srpm exist in another list)\n", $config->{verbose}, $config->{LOG}, 5); $srpm = 0 } else { log_("not found\n", $config->{verbose}, $config->{LOG}, 5) } } } if ($srpm) { $done->{rep}{$srpm} or $donesrpm = 0; $srpm[$s] = $srpm; $rpmd->[$s][1]{nosrc} or $nosrc = 0 } } } log_("put_in_rep: group $i list $list: @$rpm (@srpm) -- $curdir->[0] -- $curdir->[1] -- disc $cdnum\n", $config->{verbose}, $config->{LOG}, 4); if ($config->{nosrc} || $group->{options}{nosources} || !@{$group->{list}{$list}{srpm}} || $nosrc || $donesrpm) { ($dn) = addRPMToDiff($rpm, $rpmd, $diff, $cdnum, $group->{orderedrep}{rpm}{"$cdnum/$repname"}, $i, $relocatable_list, $curdir, $size, $rpmsize, $all_rpmsize, $relocatable_rep_num, $done, $list, $rep_num, $group) } else { if ($config->{nosrcfit} || $group->{options}{nosrcfit}) { $dn = addRPMToDiff($rpm, $rpmd, $diff, $cdnum, $group->{orderedrep}{rpm}{"$cdnum/$repname"}, $i, $relocatable_list, $curdir, $size, $rpmsize, $all_rpmsize, $relocatable_rep_num, $done, $list, $rep_num, $group); push @$nosrcfit, [$rpmd, \@srpm, $list, $i, $curdir, $cdnum] } else { my ($srpmrep, $srpmsize, $srpmok) = sourcesSizeCheck($done, $rpmd, \@srpm, $group, $groups, $size, $cdsize, $list, $cdlists, $cdnum, $all_rpmsize, $buildlist, $cds, $i, $diff); $done = $groups->[$i]{done}; if ($srpmok) { addRPMToDiff($rpm, $rpmd, $diff, $cdnum, $group->{orderedrep}{rpm}{"$cdnum/$repname"}, $i, $relocatable_list, $curdir, $size, $rpmsize, $all_rpmsize, $relocatable_rep_num, $done, $list, $rep_num, $group); $dn = addSRPMToDiff($rpmd, $done, $diff, $size, $srpmrep, $srpmsize, $curdir, \@srpm, $list, $i, $cdnum); } else { log_("WARNING: @srpm does not fit on the discs\n",1, $config->{LOG}, 2) } } if (!$dn) { foreach my $p (@$rpm) { push @{$group->{rejected}{$p}{$list}}, [ $reject_reason || "no_space", ""] } log_("WARNING: @$rpm does not fit on the disc ($size->{disc}[$cdnum] + $all_rpmsize > $cdsize->[$cdnum]) \n", $config->{verbose}, $config->{LOG}, 1) } } } return $dn } sub loop_on_lists { my ($i, $groups, $group, $groupok, $needed, $buildlist, $tobedone, $diff, $nosrcfit, $size, $cdsize, $cds, $rpmlist, $cdlists, $ok, $mark, $groupok) = @_; # # FIXME source rpms are not shared between group, it may be usefull for mutilple installation # with common source dir, so that the same source rpm is shared (but this is not so common). # my $done = $group->{done}; my $rpmd_add = sub { my ($rpm, $rpmd, $r, $list) = @_; log_("Testing $rpm\n", $config->{verbose}, $config->{LOG},7); my $d = $done->{rep}{$rpm}; my $reloc = $rpmlist->[$i]{$done->{list}{$rpm}}{$rpm}{relocatable}; if (!$d || $rpmlist->[$i]{$done->{list}{$rpm}}{$rpm}{noprovide}) { push @$rpmd, $r } elsif ($d && $reloc) { my $curdir = $group->{list}{$list}{rpm}[0]; my ($cdnum, $repname) = @$curdir; my $repnum = $group->{orderedrep}{rpm}{"$cdnum/$repname"}; if (!check_deps([ $r ], $group, $done, $rpmlist, $list, $i, $tobedone->[$i], $buildlist, $rpm, $cdnum, $repname, $needed, $repnum, $diff)) { add_relocatable_package($group, $d, $diff, $rpm, $done->{list}{$rpm}, $i, $repnum, $repname) } return 0 } 1 }; my $dn; log_("loop_on_lists: group $i (@{$group->{orderedlist}{rpm}})\n", $config->{verbose}, $config->{LOG}, 2); while (!$dn) { $groupok->[$i] = 1; foreach my $list (@{$group->{orderedlist}{rpm}}) { my $nb = @{$buildlist->[$i]{$list}} if ref $buildlist->[$i]{$list}; log_("loop_on_lists: list $list (empty $config->{list}[$list]{empty} done $config->{list}[$list]{done} nb $nb)\n", $config->{verbose}, $config->{LOG}, 3); do { $config->{list}[$list]{done} and goto end; $config->{list}[$list]{empty} and goto end; my $next; foreach my $need (@{$needed->[$i]{$list}{asap}}) { my $nb_elt = @{$buildlist->[$i]{$need->[0]}}; log_("List $list need list $need->[0] to be <= $need->[1] ($nb_elt)\n", $config->{verbose}, $config->{LOG}, 4); $nb_elt <= $need->[1] or $next = 1 } if ($next) { log_("List $list waiting\n",1, $config->{LOG},4); goto end } $needed->[$i]{$list}{asap} = []; my ($trpmd, $k, $goon, @rpmd); do { $trpmd = pop @{$buildlist->[$i]{$list}} or goto end; if (ref $trpmd->[0]) { foreach (@$trpmd) { $rpmd_add->($_->[0], \@rpmd, $_, $list) or goto end; } } else { $rpmd_add->($trpmd->[0], \@rpmd, $trpmd, $list) or goto end } } until @rpmd; $groupok->[$i] = 0; $ok = 0; my @rpm; my $all_rpmsize; my @rpmsize; foreach (@rpmd) { my $r = $_->[0]; !$r and log_("ERROR loop_on_lists: empty package @$_\n", $config->{verbose}, $config->{LOG}, 2); push @rpm, $r; log_("RPM $r (group $i list $list)\n", $config->{verbose}, $config->{LOG},6); $tobedone->[$i]{$r}{$list} = 1; $all_rpmsize += $group->{size}{$r}{$list}[0]; push @rpmsize, $group->{size}{$r}{$list}[0] } $dn = put_in_rep($i, $groups, $group, $size, \@rpmsize, $all_rpmsize, $cdsize, $needed, \@rpm, \@rpmd, $list, $cdlists, $buildlist, $diff, $cds, $done, $tobedone, $rpmlist, $nosrcfit); $done = $group->{done}; $groupok->[$i] = mark_and_check_lists($groups, $i, $needed->[$i], $diff, $buildlist->[$i], $rpmlist, $mark->[$i], $size, $cdsize, $groupok->[$i], $tobedone->[$i]) if $group->{options}{sequential} } while $group->{options}{sequential} && @{$buildlist->[$i]{$list}}; end: last if $group->{options}{sequential} && @{$buildlist->[$i]{$list}} && !$config->{list}[$list]{done} } $groupok->[$i] and $dn = 1 } return $ok } sub calc_needed_size { my ($group, $i, $needed, $needed_size, $buildlist, $rpmlist, $tobedone) = @_; my ($msg, %local_done); $msg = "calc_needed_size\n"; my $done = $group->{done}; foreach my $rep (@{$group->{replist}{rpm}}) { my ($cd, $repname, $num, $l) = @$rep; $msg .= "calc_needed_size: rep $num\n"; foreach my $list (keys %$needed) { # 2 ways here, either pre-detect that some list are done and decrease the needed # flag to the greater smaller not done rep, or just let the needed as this but # handle the packages included into this needed list, even if the considered rep # is not including this list. But later the mark_and_check list will see it and automatically schedule # the package for the previous available rep. # $l->{$list} or next; my $ok; $ok ||= $group->{listmatrix}{rpm}{$list}{$_} foreach keys %$l; $ok or next; if ($config->{list}[$list]{disc}{$cd}{$repname}{done}) { log_("calc_needed_size: rep $cd/$repname for list $list is done, ignoring\n", $config->{verbose}, $config->{LOG}, 5); next } foreach my $elt (@{$needed->{$list}{alap}[$num]}) { my $rpm = $elt->[0]; if ($done->{rep}{$rpm} && !$rpmlist->[$i]{$group->{done}{list}{$rpm}}{$rpm}{noprovide} || $local_done{$rpm}) { next } if ($group->{rejected}{$rpm}{$list}) { $msg .= "ERROR: $rpm is rejected, ignoring\n" if $config->{verbose} > 5; next } $needed_size->[$num]{fix} += $group->{size}{$rpm}{$list}[0]; $msg .= "calc_needed_size: list $list rpm $rpm size $group->{size}{$rpm}{$list}[0] (done $done->{rep}{$rpm} needed_size $num $needed_size->[$num]{fix})\n" if $config->{verbose} > 5; $local_done{$rpm} = 1; # FIXME This following code is a simplified version of check_deps. It may be overkill to use # full check_deps as anyway check_deps will be used to put the package at the end. foreach my $deps (@{$group->{pkgdeps}{$rpm}}) { if (ref $deps) { $local_done{"@$deps"} and next; $local_done{"@$deps"} = 1; my $r = choose_alt($deps, $rpmlist->[$i], $group, $cd, $num, $list, $buildlist, {}, $tobedone, $needed, 0); my ($deps_list) = find_list($config, $group, $r); if ($r != -1 && $r != 0) { next if $local_done{$r}; $needed_size->[$num]{var} += $group->{size}{$r}{$deps_list}[0]; } } else { my $pkg = $group->{depslistid}[$deps]; next if $done->{rep}{$pkg} && !$rpmlist->[$i]{$group->{done}{list}{$rpm}}{$rpm}{noprovide} || $local_done{$pkg}; $local_done{$pkg} = 1; my ($deps_list) = find_list($config, $group, $pkg); $needed_size->[$num]{var} += $group->{size}{$pkg}{$deps_list}[0]; } } } } } log_($msg, $config->{verbose}, $config->{LOG}, 4) } sub revert_to { my ($groups, $i, $p2r, $diff, $size, $buildlist) = @_; log_("revert_to: try to find $p2r (diff $diff $diff->{data} " . int @{$diff->{data}} . " $diff->{idx} " . int @{$diff->{idx}} . "\n", $config->{verbose}, $config->{LOG}, 3); foreach (@{$diff->{data}}) { $_ or next; foreach (@{$_->[5]}) { goto revert_to_ok if $_->[0] eq $p2r } } log_("ERROR revert_to: $p2r is not present in movement history\n", $config->{verbose}, $config->{LOG}, 2); return 0; revert_to_ok: log_("revert_to: $p2r found\n", $config->{verbose}, $config->{LOG}, 5); my @keep; my $idx; do { $idx = pop @{$diff->{idx}}; goto revert_to_endloop if any { $_->[0] eq $p2r } @{$diff->{data}[$idx][5]}; my $step = $diff->{data}[$idx]; if ($groups->[$i]{conflict}{$step->[1]}) { my ($curdir, $g, $list) = @$step; my $cdnum = $curdir->[0]; foreach (@{$step->[5]}) { my ($rpm, undef, undef, $rpmsize) = @$_; delete $groups->[$g]{done}{rep}{$rpm}; delete $groups->[$g]{done}{list}{$rpm}; $diff->{data}[$idx] = 0; $size->{disc}[$cdnum] -= $rpmsize; $size->{rep}{$cdnum}{$curdir->[1]}{$list} -= $rpmsize; log_("revert_to: reverting $rpm ID $idx for $p2r on cd $cdnum group $g list $list (cd size $size->{disc}[$cdnum])\n", $config->{verbose}, $config->{LOG}, 3) } push @{$buildlist->{$list}}, $step->[6] if $step->[4] == 1 } else { unshift @keep, $idx; } } while @{$diff->{idx}}; revert_to_endloop: die "FATAL revert_to: diff data are empty\n" if ! @{$diff->{data}}; push @{$diff->{idx}}, $idx, @keep; 1 } sub mark_and_check_lists { my ($groups, $i, $needed, $diff, $buildlist, $rpmlist, $mark, $size, $cdsize, $ok, $tobedone, $force_calc) = @_; my $group = $groups->[$i]; my $need_to_calc; foreach my $list (@{$group->{orderedlist}{rpm}}) { ref $buildlist->{$list} or next; log_("mark_and_check_list: group $i list $list (mark $mark)\n", $config->{verbose}, $config->{LOG}, 3); if ($config->{list}[$list]{done}) { log_("mark_and_check_lists: list $list is done, ignoring\n", $config->{verbose}, $config->{LOG}, 5); next } if (defined $mark->{cur}{$list}) { log_("mark_and_check_list: mark defined ($mark->{cur}{$list}[0])\n", $config->{verbose}, $config->{LOG},4); $need_to_calc = 1; my $m = $mark->{cur}{$list}; my $m_rpm = $m->[0]; if ($group->{done}{rep}{$m_rpm}) { log_("mark_and_check_list: $m_rpm done, deleting mark for list $list\n", $config->{verbose}, $config->{LOG},4); push @{$mark->{his}}, $m; delete $mark->{cur}{$list} } elsif ($group->{rejected}{$m_rpm}{$list}) { log_("mark_and_check_list: $m_rpm rejected, deleting mark for list $list\n", $config->{verbose}, $config->{LOG},4); delete $mark->{cur}{$list} } elsif (!@{$buildlist->{$list}}) { log_("mark_and_check_list: list $list finished and $m_rpm not done or rejected\n",1, $config->{LOG},4) } } if (!defined $mark->{cur}{$list} && @{$buildlist->{$list}}) { my $rpm; for (my $j = $#{$buildlist->{$list}}; $j >= 0; $j--) { my $t = $buildlist->{$list}[$j]; $rpm = ref $t->[0] ? $t->[0] : $t; if (!$group->{done}{rep}{$rpm->[0]} || $rpm->[1]{relocatable} || $rpm->[1]{noprovide}) { last } # this is not necessary, but when we are at it... log_("mark_and_check_list: $rpm->[0] done (list $group->{done}{list}{$rpm->[0]} rep $group->{done}{rep}{$rpm->[0]}), removing from queue\n", $config->{verbose}, $config->{LOG},4); pop @{$buildlist->{$list}} } $mark->{cur}{$list} = $rpm; log_("mark_and_check_list: marking $rpm->[0] for $list\n", $config->{verbose}, $config->{LOG},3); } } my $calc = 0; while ($need_to_calc) { $need_to_calc = 0; my $needed_size = $mark->{needed_size}; my ($need_in_rep, $av_in_rep, %done_disc) = (0, 0, {}); # # First impression would have been to check needed in reverse order, because we could imagine # that, in the current configuration, if needed 2 does not fit, for exemple, one package # is removed, needed 2 is put. But if needed does not fit at this moment, needed 2 is removed, # and needed 3 put, then needed 2 does not fit, and it is needed to revert more to make both of # them fit. # # If fact this could not happen, because if needed 3 does not fit when needed 2 has just been # put, this mean that the calc_needed_size is bogus. # foreach my $rep (@{$group->{replist}{rpm}}) { my ($cd, undef, $num) = @$rep; $need_in_rep += $needed_size->[$num]{var} + $needed_size->[$num]{fix}; if (! $done_disc{$cd}) { $av_in_rep += $cdsize->[$cd] - $size->{disc}[$cd]; $done_disc{$cd} = 1 } log_("mark_and_check_list: cd $cd rep $num need_in_rep $need_in_rep av_in_rep $av_in_rep needed_size $needed_size->[$num]{fix} disc_av_space ($cdsize->[$cd] - $size->{disc}[$cd])\n", $config->{verbose}, $config->{LOG}, 4); # Later the difference between fix and var may be used, but right now there is no code to be sure a package is on a specific disc. # if ($need_in_rep && $need_in_rep > $av_in_rep || $needed_size->[$num]{fix} > $cdsize->[$cd] - $size->{disc}[$cd]) if ($need_in_rep && $need_in_rep > $av_in_rep) { if (!$calc) { $calc = 1; $mark->{needed_size} = []; calc_needed_size($group, $i, $needed, $mark->{needed_size}, $buildlist, $rpmlist, $tobedone); $need_to_calc = 1; last } ref $mark->{his} or die "FATAL mark_and_check_list: needed packages will not fit on discs ($need_in_rep in disc $cd rep $num > $av_in_rep available)\n"; # TODO check if that is necessary or not # I think it may have been when the choose_alt bug of not rejecting packages when all the alternatives are rejected. #pop @{$mark->{his}} if @{$mark->{his}}; my $p2r; #$p2r = pop @{$mark->{his}}; while (($p2r = pop @{$mark->{his}}) && defined $p2r->[1]{needed} && $p2r->[1]{needed} <= $num) { log_("mark_and_check_list: grepping packages history $p2r->[0] needed $p2r->[1]{needed} history " . (int @{$mark->{his}}) . " \n", $config->{verbose}, $config->{LOG}, 3); } my $countdown = $mark->{cd}{$cd}{$p2r->[0]}; $p2r = pop @{$mark->{his}} while @{$mark->{his}} && $countdown-- >= 0; if (!$p2r || $mark->{cd}{$cd}{$p2r->[0]} > 2) { die "FATAL mark_and_check_list: a previous revert to put needed packages failed, cannot order packages correctly fo rep $num on disc $cd\n" } else { log_("mark_and_check_list: not enough space for needed in rep $num on disc $cd\n", $config->{verbose}, $config->{LOG}, 3); $mark->{cd}{$cd}{$p2r->[0]}++; log_("mark_and_check_list: trying to revert $p2r->[0]\n", $config->{verbose}, $config->{LOG}, 4); if (revert_to($groups, $i, $p2r->[0], $diff, $size, $buildlist)) { log_("mark_and_check_list: $p2r->[0] reverted\n", $config->{verbose}, $config->{LOG}, 3); foreach my $idx (0 .. @{$group->{orderedlist}{rpm}}) { my $list = $group->{orderedlist}{rpm}[$idx]; $needed->{$list} or next; my $elt; foreach my $tr (1 .. $num) { foreach my $elt (@{$needed->{$list}{alap}[$tr]}) { my $rpm = $elt->[0]; if ($group->{rejected}{$rpm}{$list}) { log_("ERROR: $rpm is rejected, ignoring\n", $config->{verbose}, $config->{LOG}, 6); next } push @{$buildlist->{$list}}, $elt; } } $mark->{cur}{$list} = $elt->[0]; my $l_idx = $#{$buildlist->{$list}}; next if $l_idx < 0; foreach my $tidx ($idx + 1 .. @{$group->{orderedlist}{rpm}}) { my $l = $group->{orderedlist}{rpm}[$tidx]; push(@{$needed->{$l}{asap}}, [ $list, $l_idx ]) if !$config->{list}[$list]{done} } } $calc = 0; $need_to_calc = 1; last } else { log_("ERROR mark_and_check_list: reverting to $p2r->[0] failed\n", $config->{verbose}, $config->{LOG}, 4) } } } } } return $ok } # TODO the algo is not as beautiful as it should be # ... but it is getting better # ... and better sub buildDiscs { my ($class, $groups, $buildlist, $rpmlist, $groupok, $size, $cdsize, $cdlists, $cds, $needed, $diff, $n) = @_; log_("buildDiscs\n", $config->{verbose}, $config->{LOG}, 3); $config = $class->{config}; if ($n > 1) { foreach my $i (reverse @$cds) { $size->{optimize_space}{disc}{$i} = $size->{disc}[$i]; if ($size->{disc}[$i] > $cdsize->[$i]) { my $gain = ($size->{disc}[$i] - $cdsize->[$i])/2; next if $gain < 0; optimize_space($config, $groups, $diff, $size, $cdsize, $i, $gain, $cdlists,1) } else { log_("buildDiscs: disc $i size OK $size->{disc}[$i] ($cdsize->[$i])\n", $config->{verbose}, $config->{LOG},2) } } } my ($ok, $iti); my @groupok; my (@tobedone, @nosrcfit); my @mark;# = ({}) x @$groups; $mark[$_] = {} foreach (0 .. $#$groups); updateGenericLimit($groups, $cdsize); if (!$config->{fast}) { # need to initialise needed data for (my $i = 0; $i < @$groups; $i++) { my $group = $groups->[$i]; $mark[$i]{needed_size} = []; calc_needed_size($group, $i, $needed->[$i], $mark[$i]{needed_size}, $buildlist->[$i], $rpmlist, $tobedone[$i]) } } while (!$ok) { log_("iti: " . $iti++ . "\n", $config->{verbose}, $config->{LOG},4); $ok = 1; for (my $i = 0; $i < @$groups; $i++) { my $group = $groups->[$i]; if (!$config->{fast}) { $groupok[$i] = mark_and_check_lists($groups, $i, $needed->[$i], $diff, $buildlist->[$i], $rpmlist, $mark[$i], $size, $cdsize, $groupok[$i], $tobedone[$i], !$groupok[$i]); $groupok[$i] and next; } $ok = loop_on_lists($i, $groups, $group, \@groupok, $needed, $buildlist, \@tobedone, $diff, \@nosrcfit, $size, $cdsize, $cds, $rpmlist, $cdlists, $ok, \@mark, \@groupok); } } foreach (@nosrcfit) { my ($rpmd, $srpm, $list, $i, $curdir, $cdnum) = @$_; my $group = $groups->[$i]; my $done = $group->{done}; my ($srpmrep, $srpmsize, $srpmok) = sourcesSizeCheck($done, $rpmd, $srpm, $group, $groups, $size, $cdsize, $list, $cdlists,0,0, $buildlist, $cds, $i, $diff); if ($srpmok) { addSRPMToDiff($rpmd, $done, $diff, $size, $srpmrep, $srpmsize, $curdir, $srpm, $list, $i, $cdnum); } else { log_("WARNING: @$srpm does not fit on the discs\n",1, $config->{LOG},2) } } log_("buildDiscs: rejected packages\n", $config->{verbose}, $config->{LOG},2); my $is_rejected; for (my $i; $i < @$groups; $i++) { reprocess_relocatable($groups->[$i], $cdsize, $size); $groups->[$i]{rejected} or next; foreach my $rpm (%{$groups->[$i]{rejected}}) { my $local_rejected; my $gh = $groups->[$i]{rejected}{$rpm}; my $msg; foreach my $list (keys %$gh) { $rpmlist->[$i]{$list}{$rpm}{limit} and next; $msg .= " [ list $list ] "; $local_rejected ||= any { $msg .= "$config->{rejected_options}{$_->[0]}: $_->[1],"; $_->[0] =~ /no_disc/ || $_->[0] =~ /no_space/ } @{$gh->{$list}}; chop $msg } log_("WARNING buildDisc: group $i REJECTED $rpm ($msg)\n", $config->{verbose}, $config->{LOG}, 2) if $local_rejected; $is_rejected ||= $local_rejected } } ($is_rejected) } sub reprocess_relocatable { my ($group, $cdsize, $size) = @_; # optimize number of hdlist given available space # and # add virtual media for installation my $inst_disc = $group->{installDisc}; my $inst = $config->{disc}[$inst_disc]{function}{data}{installation}; $inst->[1]{tmp_rpmsdir} = []; my %ignore; foreach my $c (@{$inst->[1]{rpmsdir}}) { my ($ls, $cdrep, $repname, $opts) = @$c; my $min; my $ok = 1; my $repnum = $group->{orderedrep}{rpm}{"$cdrep/$repname"}; log_("reprocess_relocatable: list $ls cd $cdrep repname $repname repnum $repnum\n", $config->{verbose}, $config->{LOG},3); if (defined $group->{reploc}{$repnum} && ref $group->{reploc}{$repnum}{list}) { while ($ok) { $ok = 0; log_("reprocess_relocatable: disc usage $size->{disc}[$cdrep] (disc size $cdsize->[$cdrep])\n", $config->{verbose}, $config->{LOG},6); $min = $cdsize->[$cdrep]; my $idx; for (my $i; $i < @{$group->{reploc}{$repnum}{list}}; $i++) { my $r = $group->{reploc}{$repnum}{list}[$i]; my ($new_repnum) = @$r; log_("reprocess_relocatable: new rep_num $new_repnum size $group->{reploc}{$repnum}{size}{$new_repnum} old $group->{reploc}{$repnum}{old}{$new_repnum}\n", $config->{verbose}, $config->{LOG},6); if (!$ignore{$new_repnum} && $group->{reploc}{$repnum}{old}{$new_repnum} && $group->{reploc}{$repnum}{size}{$new_repnum} < $min) { log_("reprocess_relocatable: min $min idx $i\n", $config->{verbose}, $config->{LOG},6); $min = $group->{reploc}{$repnum}{size}{$new_repnum}; $idx = $i } } if ($size->{disc}[$cdrep] + $min < $cdsize->[$cdrep]) { $ok = 1; $size->{disc}[$cdrep] += $min; my $prev = $group->{reploc}{$repnum}{list}[$idx-1][0] if $idx > 0; my $current = $group->{reploc}{$repnum}{list}[$idx][0]; my $next = $group->{reploc}{$repnum}{list}[$idx+1][0] if $idx < @{$group->{reploc}{$repnum}{list}}; if (!$prev) { $prev = $next; $next = 0 } my $curdir = $group->{reploc}{$repnum}{curdir}{$prev}; my $list = $group->{reploc}{$repnum}{newlist}{$prev}; my $reallist = $group->{reploc}{$repnum}{reallist}{$prev}; log_("reprocess_relocatable: aggregating $current and $next with $prev list $list reallist $reallist\n", $config->{verbose}, $config->{LOG},6); foreach my $rep ($current, $next) { $rep or next; foreach my $diff_data (@{$group->{reploc}{$repnum}{diff}{$rep}}) { log_("reprocess_relocatable: updating diff for rep $rep rpm $diff_data->[5][0][0] (cd $curdir->[0] rep $curdir->[1])\n", $config->{verbose}, $config->{LOG},6); $diff_data->[0] = $curdir; $diff_data->[2] = $list; $diff_data->[3] = $prev; $diff_data->[8] = $reallist; push @{$group->{reploc}{$repnum}{diff}{$prev}}, $diff_data } } $group->{reploc}{$repnum}{list} = [ grep { $_->[0] != $current && $_->[0] != $next } @{$group->{reploc}{$repnum}{list}} ] } } my $fct = $config->{disc}[$cdrep]{function}{data}{generic}{$repname}[0]; foreach my $r (@{$group->{reploc}{$repnum}{list}}) { my ($new_repnum, $cd, $repname, $newlist) = @$r; log_("reprocess_relocatable: new rep_num $new_repnum on cd $cd rep $repname list $newlist\n", $config->{verbose}, $config->{LOG},3); push @{$inst->[1]{tmp_rpmsdir}}, [ $newlist, $cd, $repname ]; push @{$fct->[1]{lists}}, $newlist } } push @{$inst->[1]{tmp_rpmsdir}}, $c } my $i; foreach my $r (@{$inst->[1]{tmp_rpmsdir}}) { log_("reprocess_relocatable: testing $r->[1]/$r->[2] ($i)\n", $config->{verbose}, $config->{LOG},3); if (defined $inst->[1]{boot_medium} && $r->[1] == $inst_disc && $i) { log_("reprocess_relocatable: boot disc is $i\n", $config->{verbose}, $config->{LOG},3); $inst->[1]{boot_medium} = $i + 1; last } $i++ } } sub processDiff { my ($class, $groups, $diff, $discsFiles) = @_; my (@cd, @action); my %new; my $prev = $diff->{previous_idx} || {}; foreach (@{$diff->{idx}}) { push @{$action[1]}, $_ if !$prev->{$_}; $new{$_} = 1 } foreach (keys %$prev) { push @{$action[2]}, $_ if !$new{$_} } foreach my $op (2,1) { foreach my $idx (@{$action[$op]}) { my $d = $diff->{data}[$idx]; if (!$d) { log_("ERROR processDiff: THIS MUST NOT HAPPEN action is null ($d) op $op idx $idx\n", $config->{verbose}, $config->{LOG}); next } my ($curdir, $grp, $list, undef, undef, $data, undef, undef, $reallist) = @$d; my $cd = $curdir->[0]; foreach my $ent (@$data) { my $rpm = $ent->[0]; if (!$rpm) { foreach (@$ent) { if (ref $_) { log_("ERROR processDiff: @$_\n", $config->{verbose}, $config->{LOG},2) } else { log_("ERROR processDiff: $_\n", $config->{verbose}, $config->{LOG},2) } } } $rpm or next; my $source = $groups->[$grp]{size}{$rpm}{$reallist || $list}[1]; log_("LOG disc $cd/$curdir->[1] list $list ($reallist) group $grp: ($op) $rpm ($source)\n", $config->{verbose}, $config->{LOG},3); push @{$cd[$cd]{$curdir->[1]}{$list}{$source}}, [$op, "$groups->[$grp]{urpm}{rpmkey}{rpm}{$rpm}.rpm"]; if ($op == 1) { $discsFiles->[$cd]{$curdir->[1]}{$list}{$groups->[$grp]{urpm}{rpmkey}{rpm}{$rpm}} = $source } elsif ($op == 2) { delete $discsFiles->[$cd]{$curdir->[1]}{$list}{$groups->[$grp]{urpm}{rpmkey}{rpm}{$rpm}} } } } } my %new_diff; # clear diff foreach my $idx (@{$diff->{idx}}) { my $nidx = push @{$new_diff{data}}, $diff->{data}[$idx]; push @{$new_diff{idx}}, $nidx - 1; $new_diff{previous_idx}{$nidx - 1} = 1 } return \@cd, \%new_diff } 1 # Changelog # # 2002 02 21 # change false $j comparaison to $depsdisc in buildDisc to new $thisorderrep value. # # 2002 03 03 # new limit option handling. # add updateGenericSoft function # add testSoftLimit function # update size to check rep size # # 2002 03 08 # fix autoMode CD adding # # 2002 03 13 # better selection of alternatives in multi-list to take the one in the first lists. # # 2002 03 14 # add sources new sources handling method # in nosrcfit mode sources are added afterwards # # 2002 03 19 # add prelist in geList for cdcom, will be useful for oem too I guess. # # 2002 05 02 # add_one_disc: add separate mode for sources mode # # 2002 05 09 # add graft structure for md5 and graft point handling # # 2002 05 13 # fix a tricky bugs in build_list about fentry shared and not recreated for each packages. # # 2002 06 01 # use perl-URPM # # 2002 06 15 # new diff mode, global, shared between disc and group, only one table. # # 2002 08 16 # new diff_idx table to sort diff data # # 2002 08 24 # optimize_space first version, still need to handle correctly needed and more advanced optimization methods. # # 2002 09 18 # optimize_space work, fixes and updates. # # 2002 10 25 # fix needed assignation pb in closeRPMslist # # 2004 05 27 # separate List.pm into List.pm and Build.pm # # 2004 05 28 # move find_list to tools as it is used in both Build and List