package Mkcd::List; my $VERSION = '3.0.0'; use strict; use File::NCopy qw(copy); use File::Path; use URPM qw(ranges_overlap); use Mkcd::Package qw(rpmVersionCompare); use Mkcd::Tools qw(log_ find_list); use Mkcd::Optimize qw(optimize_space get_pkgs_deps); use MDK::Common qw(any if_); my $MIN_CHUNK = 0.0001; =head1 NAME List - mkcd module =head1 SYNOPSYS require Mkcd::List; =head1 DESCRIPTION C include the mkcd packages list functions. =head1 SEE ALSO mkcd =head1 COPYRIGHT Copyright (C) 2000,2001,2002,2003,2004 Mandrakesoft This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. =cut my $config; sub new { my ($class, $conf) = @_; $config = $conf; bless { config => $config, }, $class; } sub getDoneList { my ($config, $group, $listnumber, $discsFiles) = @_; my $topdir = "$config->{topdir}/build/$config->{name}"; my $add_rpm = sub { my ($d, $rpm, $cd, $rep, $type) = @_; #$rpm =~ /($test)\.rpm$/ or return; $rpm =~ /(.*)\.rpm$/ or return; #$rpm =~ /src\.rpm$/ and return; my $rpm = $group->{urpm}{rpmkey}{key}{$1}; $group->{done}{rep}{$rpm} = $group->{orderedrep}{$type}{"$cd/$rep"}; $group->{done}{list}{$rpm} = $listnumber; $discsFiles->[$cd]{$rep}{$listnumber}{$1} = $d }; my $read_dir = sub { my ($dir, $cd, $rep, $type) = @_; my $RPMS; if (!opendir $RPMS, $dir) { log_("WARNING getGroupReps: cannot open $dir\n",1, $config->{LOG},0); return } foreach (readdir $RPMS) { if (-d "$dir/$_") { opendir my $D, "$dir/$_"; foreach my $r (readdir $D) { $add_rpm->("$dir/$_", $r, $cd, $rep, $type) } } else { $add_rpm->($dir, $_, $cd, $rep, $type) } } }; foreach my $type ('rpm', 'srpm') { foreach my $curdir (@{$group->{list}{$listnumber}{$type}}) { my ($cd, $rep) = @$curdir; my $path = $config->{disc}[$cd]{function}{data}{dir}{$rep}[1]{reploc}; my $dir = "$topdir/$cd/$path"; log_("getDoneList: $listnumber disc $cd rep $rep\n", $config->{verbose}, $config->{LOG},2); if ($config->{nolive}) { if ($config->{list}[$listnumber]{packages}) { foreach my $ent (@{$config->{list}[$listnumber]{packages}}) { foreach my $type (keys %$ent) { my $rpm_tab = $ent->{$type}; log_("getDoneList: @$rpm_tab\n",1, $config->{LOG},3); foreach my $dir (@$rpm_tab) { $read_dir->($dir, $cd, $rep, $type) } } } } else { log_("ERROR getDoneList: could not find data for $listnumber disc $cd rep $rep\n", $config->{verbose}, $config->{LOG},0); } } elsif (-d $dir) { $read_dir->($dir, $cd, $rep, $type); } else { log_("ERROR getDoneList: could not find data for $listnumber disc $cd rep $rep\n", $config->{verbose}, $config->{LOG},0); next } $config->{list}[$listnumber]{disc}{$cd}{$rep}{done} = 1 } } foreach my $d (@{$config->{list}[$listnumber]{virtual}}) { my $cd = $d->{disc}; my $path = $d->{path}; my $rep = $d->{repname}; my $dir = "$topdir/$cd/$path"; log_("getDoneList: virtual disc $cd path $path in $dir\n",1, $config->{LOG},2); foreach my $list_rep (@{$group->{rep}{$listnumber}}) { $list_rep->{rpm}{$dir} or next; foreach (@{$list_rep->{rpm}{$dir}}) { my $rpm = $group->{urpm}{rpmkey}{key}{$_}; $group->{done}{rep}{$rpm} = $group->{orderedrep}{rpm}{"$cd/$rep"}; $group->{done}{list}{$rpm} = $listnumber; $discsFiles->[$cd]{$rep}{$listnumber}{$_} = $dir } } $config->{list}[$listnumber]{disc}{$cd}{$rep}{done} = 1; } } sub read_filter_list { my ($file, $listnumber, $config, $filelist, $norpmsrate) = @_; log_("read_filter_list: FILE LIST listnumber $listnumber ($file)\n", $config->{verbose}, $config->{LOG},2); my $A; if (! open $A, $file) { log_("ERROR read_filter_list: cannot open $file, ignoring\n",1, $config->{LOG}); return } #local $_; my $line; while (<$A>) { $line++; s/#.*//; next if !$_ || /^\s*$/; my ($name, $options) = /\s*(\S+)\s*(.*)/; my @options = split ',', $options; log_("FILESLIST: $_ -> $name options @options\n", $config->{verbose}, $config->{LOG},3); my %opt; foreach (@options) { s/^\s*//g; /norpmsrate/ and push @$norpmsrate, $name and next; if (!/^(?:(?:nosrc|section|noalternatives|regexp|ignore|nodeps|force|limit|exclude)|(rate|notinrep|inrep|rpmsrate|needed|section) (\d+))$/) { log_("WARNING: getList: $_: unknown option\n",1, $config->{LOG}); next } $_ = $1 || $_; if (/^inrep/) { $opt{needed} = $2 } $opt{$_} = $2 || 1; } if ($name =~ /[*|]/ && !$opt{regexp}) { die "FATAL read_filter_list $file line $line: \"$name $options\" could not have * or | if regexp keyword is not used\n" } log_("Adding $name -- " . join(' ', keys %opt) . "\n", $config->{verbose}, $config->{LOG},4); push @{$filelist->{$listnumber}}, [ $name, \%opt ]; } close $A } sub getList { my ($class, $group, $discsFiles) = @_; my $config = $class->{config}; my %filelist; my @norpmsrate; log_("getList: group list " . join(' ', (keys %{$group->{list}})) . "\n", $config->{verbose}, $config->{LOG},1); foreach my $listnumber (keys %{$group->{list}}) { my $done = $config->{list}[$listnumber]{done}; $done and getDoneList($config, $group, $listnumber, $discsFiles); log_("getList: FILE LIST listnumber $listnumber\n", $config->{verbose}, $config->{LOG},2); if ($config->{list}[$listnumber]{filelist} || $config->{list}[$listnumber]{prelist}) { foreach my $file (@{$config->{list}[$listnumber]{filelist}}) { read_filter_list($file, $listnumber, $config, \%filelist, \@norpmsrate); } foreach my $p (@{$config->{list}[$listnumber]{prelist}}) { log_("Prelist Adding $p->[0] -- " . join(' ', keys %{$p->[1]}) . "\n", $config->{verbose}, $config->{LOG},3); $p->[1]{norpmsrate} and push @norpmsrate, $_->[0] and next; push @{$filelist{$listnumber}}, $p } } else { if (!$done && $config->{list}[$listnumber]{auto}) { push @{$filelist{$listnumber}}, [ "INSTALL", { section => 1, force => 1 } ]; push @{$filelist{$listnumber}}, [ "SYSTEM", { section => 1, force => 1 } ]; push @{$filelist{$listnumber}}, [ "kernel-(smp-|)[0-9].*", { regexp => 1, force => 1 } ]; push @{$filelist{$listnumber}}, [ "kernel.*linus", { regexp => 1, noalternatives => 1 } ]; push @{$filelist{$listnumber}}, [ ".*", { regexp => 1 } ] } else { log_("getList: FILE LIST listnumber $listnumber defaulting to .* regexp\n", $config->{verbose}, $config->{LOG},2); push @{$filelist{$listnumber}}, [ ".*" , { regexp => 1 } ] } } my $listdone = 1; foreach my $r (@{$group->{list}{$listnumber}{rpm}}) { my ($cd, $rep, $repopt, $opt) = @$r; log_("getList: searching for done rep ($cd/$rep) dup $opt->{dup}\n", $config->{verbose}, $config->{LOG},2); if ($config->{list}[$listnumber]{disc}{$cd}{$rep}{done}) { if (!$opt->{dup}) { foreach my $rpmkey (keys %{$discsFiles->[$cd]{$rep}{$listnumber}}) { my $rpm = $group->{urpm}{rpmkey}{key}{$rpmkey}; $group->{done}{rep}{$rpm} = $group->{orderedrep}{rpm}{"$cd/$rep"}; $group->{done}{list}{$rpm} = $listnumber; log_("getList: $rpm in $cd/$rep -> $group->{done}{rep}{$rpm} (noprovide $opt->{noprovide} relocatable $opt->{relocatable} update $opt->{update})\n", $config->{verbose}, $config->{LOG},3); push @{$filelist{$listnumber}}, [ $rpm, { done => $group->{done}{rep}{$rpm}, regexp => 1, udpate => $opt->{update}, noprovide => $opt->{noprovide}, relocatable => $opt->{relocatable} } ]; } } } else { $listdone = 0 } } if ($listdone) { log_("getList: setting list $listnumber as done\n", $config->{verbose}, $config->{LOG},2); $config->{list}[$listnumber]{done} = 1 } } (\%filelist, \@norpmsrate) } # # compute individual scoring (max_size*(rpmsrate+1)*rpmsrate_factor/(size*size_factor)) # then add dependencies sons score ( score + deps_factor*(sons_score) # # special rpmsrate groups score could be added in the rpmsrate value # # FIXME current scoring rules make size only significant for equaly dependent packages, # dependencies get far more importance for packages a lot of packages depend on. # # Size scoring could be added afterwards, but this will break the autodeps created with # this scheme. # # TODO # add scoring rules to include srpm size in score. # # sub scoreList { my ($class, $group) = @_; my $scoreweight = $group->{score}; my $urpm = $group->{urpm}; my $rpmsrate = $group->{rpmsrate}; my $maxsize = $group->{maxsize} || 1; $scoreweight ||= [1,1,1]; log_("scoreList: SCORE for group: @$scoreweight\n", $config->{verbose}, $config->{LOG},2); log_("scoreList: Individual scoring\n", $config->{verbose}, $config->{LOG},2); my ($sf, $i, $total); my (@min, @max); if ($scoreweight->[1]) { (@min,@max) = (($maxsize*$scoreweight->[0]*6/($scoreweight->[1]*1),0), (0,0)) } else { (@min,@max) = (($maxsize*$scoreweight->[0]*6,0), (0,0)) } my @specialdeps; foreach (keys %{$urpm->{rpm}}) { # print "INFO KEYS $_\n"; my ($ratekey) = /(.*)-[^-]+-[^-]+\.[^.]+$/; # FIXME take the bigger size when package appears in multiple lists my $size; foreach my $list (keys %{$group->{size}{$_}}) { $size = $group->{size}{$_}{$list}[0] if $size < $group->{size}{$_}{$list}[0] } $size or log_("WARNING scoreList: $_ has zero size\n",1, $config->{LOG}); my $s; my $rate = $group->{brokendeps}{$_} ? 0 : (defined $group->{pkgrate}{$_} ? $group->{pkgrate}{$_} : $rpmsrate->[0]{$ratekey}); if ($scoreweight->[1]) { $sf = ($size*9)/$maxsize + 1; # from 1 to 10 $s = $scoreweight->[0]*($rate + 1)/($scoreweight->[1]*$sf); } else { $s = $scoreweight->[0]*($rate + 1); } $group->{scorelist}{$_} = $s; $s < $min[0] and @min = ($s, $_); $s > $max[0] and @max = ($s, $_); $total+=$s; $i++ } $i and log_("scoreList: minimal $min[0] ($min[1]), maximal $max[0] ($max[1]), average " . $total/$i . "\n", $config->{verbose}, $config->{LOG},3); 1 } sub autodeps { my ($class, $group, $rpmlist) = @_; my $scoredeps = $group->{score}[2]; if (!$scoredeps) { log_("autodeps: deps score is null, bypassing autodeps\n",1, $config->{LOG},1); return 1 } log_("autodeps: compute reversed depslist.ordered ($scoredeps)\n", $config->{verbose}, $config->{LOG},2); my $revDeps = $group->{revdeps}; my %rpm; foreach my $k (values %$rpmlist) { foreach (keys %$k) { $rpm{$_} = $k->{$_} } } # FIXME this algo is not correct ref $group->{urpm}{depslist} or return 0; for (my $i = @{$group->{urpm}{depslist}} - 1; $i >= 0; $i--) { my $rpm = $group->{depslistid}[$i]; if (!$rpm{$rpm}) { #log_("autodeps: ignoring $rpm\n",1, $config->{LOG}); #push @{$group->{rejected}{$rpm}}, [ "autodeps", $1 ]; next } if ($rpm{$rpm}{ignore}) { log_("autodeps: $rpm has ignore flag, do not add deps score\n", $config->{verbose}, $config->{LOG},3); next } foreach (@{$revDeps->[$i]}) { $group->{scorelist}{$rpm} += $scoredeps * $group->{scorelist}{$group->{depslistid}[$_]}; } } 1 } sub reverseDepslist { my ($class, $group, $rpmfile) = @_; my $urpm = $group->{urpm}; my $depslist = $urpm->{depslist}; if (!$depslist) { log_("WARNING reverseDepslist: no depslist found\n", $config->{verbose}, $config->{LOG},3); return () } my $locales = $group->{lang}; my (@revdeps, %skip); log_("reverseDepslist\n", $config->{verbose}, $config->{LOG},2); # urpmi remove basesystem require from the depslist my %cache; my $get_name_version = sub { my ($n, $s) = @_; # perl need major checking #if ($n eq 'perl') { $s and ($s =~ /:/ or $s =~ s/([>=<]+) /$1 0:/); # Now we should have all the epoch fixed. # } else { # # Some deps are broken and as a consequence using the major rejects some packages that shouldn't # #$s and ($s =~ /:/ or $s =~ s/([>=<]+) /$1 0:/); # $s =~ /:/ and $s =~ s/ \d+:/ /; # } $n,$s }; for (my $i; $i < @$depslist; $i++) { my $d = $depslist->[$i]; foreach my $req ($d->requires) { my ($n, $s) = $req =~ /^([^\s\[]*)(?:\[\*\])?\[?([^\s\]]*\s*[^\s\]]*)/; ($n, $s) = $get_name_version->($n, $s); push @{$cache{requires}[$i]}, [$n, $s]; $cache{all_requires}{$n} = 1 } } for (my $i; $i < @$depslist; $i++) { my $d = $depslist->[$i]; $cache{ver}[$i] = $d->version; $cache{verrel}[$i] = sprintf "%s-%s", $d->version, $d->release; $cache{name}[$i] = sprintf "%s-%s.%s", $d->name, $cache{verrel}[$i], $d->arch; foreach my $prov ($d->provides, @{$urpm->{files}{$d->fullname}}) { my ($deps_n, $deps_s) = $prov =~ /^([^\s\[]*)(?:\[\*\])?\[?([^\s\]]*\s*[^\s\]]*)/; #$deps_s and ($deps_s =~ /:/ or $deps_s =~ s/([>=<]+) /$1 0:/); ($deps_n, $deps_s) = $get_name_version->($deps_n, $deps_s); $cache{all_requires}{$deps_n} or next; push @{$cache{provides}{$deps_n}{$i}}, $deps_s } } # FIXME could not clean the {files} as some group may use it afterwards # some memory cleanup # delete $urpm->{files}; my $count; for (my $i; $i < @$depslist; $i++) { my $d = $depslist->[$i]; my $rpm = $cache{name}[$i]; $group->{depslistid}[$i] = $rpm; my %rev; my @deps = map { split '\|', $_ } split ' ', $urpm->{deps}[$i]; # urpm has an optimization which removes most requires/provides in depslist regarding basesystem packages. @deps or next; log_("reverseDepslist: $i deps for $rpm\n", $config->{verbose}, $config->{LOG}, 5); if (my @missed_deps = map { m/NOTFOUND_(\S*)/ and $1 } grep { m/NOTFOUND_/ } @deps) { foreach my $list (keys %{$group->{size}{$rpm}}) { if (!$group->{options}{nodeps} && !$class->{config}{nodeps} && !(defined $rpmfile->{$list}{$rpm} && $rpmfile->{$list}{$rpm}{nodeps})) { $skip{$i} = 1; $group->{brokendeps}{$rpm} = 2; push @{$group->{rejected}{$rpm}{$list}}, [ "deps", "@missed_deps" ]; } log_("WARNING reverseDepslist: $rpm has unresolved dependencies (@missed_deps)\n", $config->{verbose}, $config->{LOG}, 1); next } } my $first = 1; my $msg; foreach my $req (@{$cache{requires}[$i]}) { my ($n, $s) = @$req; my @pkg; #log_("reverseDepslist: $rpm requires $n $s\n", $config->{verbose}, $config->{LOG}, 1); my $own_provide; my $add_msg; foreach (keys %{$cache{provides}{$n}}) { $msg = ''; my $deps_v = $cache{verrel}[$_]; my $deps_rpm = $cache{name}[$_]; foreach my $deps_s (@{$cache{provides}{$n}{$_}}) { if ($deps_s) { if (URPM::ranges_overlap($deps_s, $s)) { if ($_ == $i) { $own_provide = 1; last } $add_msg .= "reverseDepslist: adding $deps_rpm ($n $deps_s for $s)\n"; push @pkg, $_; } else { $msg .= " $deps_rpm provides $n $deps_s but $rpm needs $n $s" } } else { if ($_ == $i) { $own_provide = 1; last } $add_msg .= "reverseDepslist: adding $deps_rpm ($n)\n"; push @pkg, $_; } } if ($first) { if ($locales && $group->{depslistid}[$_] =~ /locales-([^-]+)-[^-]+-[^-]+\.[^.]+/) { if (!$locales->{$1}) { log_("reverseDepslist: locale $1 ($group->{depslistid}[$_]) skipped for $rpm\n", $config->{verbose}, $config->{LOG}, 2); $skip{$i} = 1; !$group->{brokendeps}{$rpm} and $group->{brokendeps}{$rpm} = 1 } } $skip{$_} or push @{$revdeps[$_]}, $i } last if $own_provide } $first = 0; next if ($own_provide); if (@pkg == 0) { $skip{$i} = 1; $group->{brokendeps}{$rpm} = 2; log_("WARNING reverseDepslist: rejecting $rpm on $n\n", $config->{verbose}, $config->{LOG}, 4); foreach my $list (keys %{$group->{size}{$rpm}}) { push @{$group->{rejected}{$rpm}{$list}}, [ "deps", "$n $msg" ]; } last } elsif (@pkg == 1) { log_($add_msg, $config->{verbose}, $config->{LOG}, 8); push @{$group->{pkgdeps}{$rpm}}, $pkg[0] } else { log_($add_msg, $config->{verbose}, $config->{LOG}, 8); push @{$group->{pkgdeps}{$rpm}}, \@pkg } } } return \@revdeps; } # FIXME this function is broken sub check_version { my ($d, $deps_d, $rpm, $special_require) = @_; my ($msg, $real_ok, $ok); $ok = 0; my $f = 1; my $deps_v = sprintf "%s-%s", $deps_d->version, $deps_d->release; my $deps_rpm = sprintf "%s-%s.%s", $deps_d->name, $deps_v, $deps_d->arch; $rpm ||= sprintf "%s-%s-%s.%s", $d->name, $d->version, $d->release, $d->arch; foreach my $req ($d->requires) { my ($n, $s) = $req =~ /^([^\s\[]*)(?:\[\*\])?\[?([^\s\]]*\s*[^\s\]]*)/; # perl needs major checking $n eq 'perl' and $ok = 1 and next; # Some deps are broken and as a consequence using the major rejects some packages that shouldn't #$s and ($s =~ /:/ or $s =~ s/([>=<]+) /$1 0:/); $s =~ /:/ and $s =~ s/ \d+:/ /; if ($n && $s) { # FIXME there is an approximation here, a package which provides a 1.3 and b 2.4 could satisfy a 1.3 but not b 2.5 for example # however it will be rejected anyway. Otherwize any require should be checked when building the CDs, and this would be overkill. foreach my $prov ($deps_d->provides) { my ($deps_n, $deps_s) = $prov =~ /^([^\s\[]*)(?:\[\*\])?\[?([^\s\]]*\s*[^\s\]]*)/; #$deps_s and ($deps_s =~ /:/ or $deps_s =~ s/([>=<]+) /$1 0:/); $deps_s =~ /:/ and $deps_s =~ s/ \d+:/ /; $deps_s ||= $s =~ /-/ ? "= $deps_v" : "= " . $deps_d->version; if ($deps_n && $deps_s) { if ($deps_n eq $n) { if (URPM::ranges_overlap($deps_s, $s)) { $real_ok = 1 if !$special_require || $special_require eq $n; $ok = 1 & $f } else { my $t = "$deps_rpm provides $deps_n $deps_s but $rpm needs $n $s"; $msg .= " $t"; $ok = 0; $f = 0 } } } } } else { foreach my $prov ($deps_d->provides) { my ($deps_n) = $prov =~ /^([^\s\[]*)(?:\[\*\])?\[?([^\s\]]*\s*[^\s\]]*)/; if ($deps_n eq $n) { $real_ok = 1 if !$special_require || $special_require eq $n; $ok = 1 & $f } } } } ($ok, $msg, $real_ok) } sub closeRpmsList { my ($group, $rpmfile) = @_; my $n = 1; my %done; my %doneName; my %alternatives; my $inst_disc = $group->{installDisc}; my $inst = $config->{disc}[$inst_disc]{function}{data}{installation}; while ($n) { $n = 0; foreach my $listnumber (@{$group->{orderedlist}{rpm}}) { foreach my $rpm (keys %{$rpmfile->{$listnumber}}) { # FIXME if the different packages have different deps, old packages may be better. # this should be done in buildDisc, or better packages have to be selected here. if (!$group->{options}{dup}) { my ($name, $version, $release, $arch) = $rpm =~ /^(.*)-([^-]+)-([^-]+)\.([^.]+)$/; if ($doneName{$name}) { if (!($doneName{$name}[0] eq "$version-$release.$arch")) { log_("WARNING closeRpmsList: $name-$version-$release.$arch duplicated with $doneName{$name}[0]\n", $config->{verbose}, $config->{LOG}, 1); my ($v, $r, $a) = @{$doneName{$name}[1]}; my $todel; my $vers; my $ret = rpmVersionCompare($rpm, "$name-$v-$r.$a"); if ($ret < 0) { $todel = $rpm; $vers = [$v, $r, $a] } elsif ($ret > 0) { $todel = "$name-$v-$r.$a"; $vers = [$version, $release, $arch] } else { log_("ERROR closeRpmsList: oops, something not possible happened in duplicate version comparaison ($rpm and $name-$v-$r.$a)\n",1, $config->{LOG}); } if ($todel) { log_("closeRpmsList: deleting $todel\n", $config->{verbose}, $config->{LOG},2); $doneName{$name} = [ "$vers->[0]-$vers->[1].$vers->[2]", $vers ]; if (!$inst->[1]{dup}) { $group->{brokendeps}{$todel} = 3; push @{$group->{rejected}{$todel}{$listnumber}}, [ "old_version", "$name-$vers->[0]-$vers->[1].$vers->[2]" ]; } delete $rpmfile->{$listnumber}{$todel} if !$rpmfile->{$listnumber}{$rpm}{done}; log_("closeRpmsList: resetting alternatives\n", $config->{verbose}, $config->{LOG},4); foreach (keys %alternatives) { $alternatives{$_}[0] eq $todel and delete $alternatives{$_} } $todel eq $rpm and next } $n = 1 } } else { $doneName{$name} = [ "$version-$release.$arch", [$version, $release, $arch] ] } } if ($group->{brokendeps}{$rpm} == 2 || $group->{brokendeps}{$rpm} == 3) { log_("closeRpmsList: deleting $rpm (list $listnumber)\n", $config->{verbose}, $config->{LOG},2); delete $rpmfile->{$listnumber}{$rpm}; $n = 1; next } $done{$rpm} and next; $rpmfile->{$listnumber}{$rpm}{nodeps} and next; my $needed; $needed = $group->{maxrep}{rpm} if $rpmfile->{$listnumber}{$rpm}{force}; $needed ||= $rpmfile->{$listnumber}{$rpm}{needed}; $needed ||= $rpmfile->{$listnumber}{$rpm}{done}; # $needed and log_("closeRpmsList: $rpm needed set to $needed (force $rpmfile->{$listnumber}{$rpm}{force} done $rpmfile->{$listnumber}{$rpm}{done} needed $rpmfile->{$listnumber}{$rpm}{needed})\n", $config->{verbose}, $config->{LOG}); foreach (@{$group->{pkgdeps}{$rpm}}) { if (m/NOTFOUND_(.*)/) { log_("ERROR closeRpmsList: $1 not provided\n", $config->{verbose}, $config->{LOG},1); next } my $rpmdep; my $rpmdeplist; my $specialrpmdep; if (ref $_) { if ($alternatives{"@$_"}) { ($rpmdep, $rpmdeplist) = @{$alternatives{"@$_"}}; log_("closeRpmsList: $rpm taking alternatives from already selected package ($rpmdep $rpmdeplist)\n", $config->{verbose}, $config->{LOG}, 4) } my $alt_name; if (! defined $rpmfile->{$rpmdeplist}{$rpmdep} || ! ref $rpmfile->{$rpmdeplist}{$rpmdep}) { foreach my $testalt (1,0) { ($rpmdep, $rpmdeplist) = (undef,undef); # FIXME this is wrong, package can come from any list my @score = ($group->{maxlist}{rpm}, int @{$group->{list}{$listnumber}{rpm}}, $group->{maxsize}, $group->{maxrep}{rpm}); my @specialscore = (int @{$group->{list}{$listnumber}{rpm}}, $group->{maxsize}); log_("closeRpmsList: $rpm @$_ (maxscore @score) alternative\n", $config->{verbose}, $config->{LOG}, 4); $alt_name = ""; foreach (@$_) { my $pkg = $group->{depslistid}[$_]; $alt_name .= "$pkg "; log_("closeRpmsList: trying $pkg (brokendeps $group->{brokendeps}{$pkg})\n", $config->{verbose}, $config->{LOG}, 4); $group->{brokendeps}{$pkg} == 2 and next; $group->{brokendeps}{$pkg} == 3 and next; my ($pkglist) = find_list($config, $group, $pkg, $listnumber); if (!$pkglist) { log_("closeRpmsList: $pkg list could not be used for $rpm dependencies\n", $config->{verbose}, $config->{LOG}, 4); next } log_("closeRpmsList: list $pkglist\n", $config->{verbose}, $config->{LOG}, 4); if (defined $rpmfile->{$pkglist}{$pkg}) { #$rpmfile->{$pkglist}{$pkg}{limit} and next; $testalt and $rpmfile->{$pkglist}{$pkg}{noalternatives} and next; #$testalt and $rpmfile->{$pkglist}{$pkg}{noprovide} and next; } my $rep = $group->{size}{$pkg}{$pkglist}[2]; my $s = $group->{size}{$pkg}{$pkglist}[0]; my $l = $group->{listsort}{$pkglist}{rpm}; my $d = $group->{done}{rep}{$pkg}; log_("\t$pkg ($l, $rep, $s, $d) (@score)\n", $config->{verbose}, $config->{LOG}, 5); # also put an alternative from this list if ($pkglist == $listnumber) { if ($rep < $specialscore[1]) { @specialscore = ($rep, $s); $specialrpmdep = $pkg; } elsif ($rep == $specialscore[1] && $s < $specialscore[2]) { @specialscore = ($rep, $s); $specialrpmdep = $pkg; } } if ($d) { if ($d < $score[3]) { @score = ($l, $rep, $s, $d); $rpmdep = $pkg; $rpmdeplist = $pkglist; log_("5 $rpmdep -- $rpmdeplist -- $l, $rep, $s, $d\n", $config->{verbose}, $config->{LOG}, 6); } } elsif ($l < $score[0]) { @score = ($l, $rep, $s, $d); $rpmdep = $pkg; $rpmdeplist = $pkglist; log_("1 $rpmdep -- $rpmdeplist -- $l, $rep, $s, $d\n", $config->{verbose}, $config->{LOG}, 6); } elsif ($l == $score[0]) { if ($pkglist == $listnumber) { if ($rep < $score[1]) { @score = ($l, $rep, $s, $d); $rpmdep = $pkg; $rpmdeplist = $pkglist; log_("2 $rpmdep -- $rpmdeplist -- $l, $rep, $s, $d\n", $config->{verbose}, $config->{LOG}, 6); } elsif ($rep == $score[1] && $s < $score[2]) { @score = ($l, $rep, $s, $d); $rpmdep = $pkg; $rpmdeplist = $pkglist; log_("3 $rpmdep -- $rpmdeplist -- $l, $rep, $s, $d\n", $config->{verbose}, $config->{LOG}, 6); } } elsif ($s < $score[2]) { @score = ($l, $rep, $s, $d); $rpmdep = $pkg; $rpmdeplist = $pkglist; log_("4 $rpmdep -- $rpmdeplist -- $l, $rep, $s, $d\n", $config->{verbose}, $config->{LOG}, 6); } } } last if $rpmdep && $rpmdeplist } if ($rpmdep && $rpmdeplist) { log_("\tResult:\t$rpmdep\n", $config->{verbose}, $config->{LOG}, 4); $alternatives{"@$_"} = [ $rpmdep, $rpmdeplist ] } else { log_("WARNING: $rpm has unresolved or excluded dependencies ($alt_name), removed\n", $config->{verbose}, $config->{LOG}, 1); log_("closeRpmsList: deleting $rpm (list $listnumber)\n", $config->{verbose}, $config->{LOG}, 2); delete $rpmfile->{$listnumber}{$rpm}; $n = 1; $group->{brokendeps}{$rpm} = 2; push @{$group->{rejected}{$rpm}{$listnumber}}, ["deps_rejected", (join ' ', map { $group->{depslistid}[$_] } @$_)]; last } } } else { # TODO verify that there is no need to do $rpmfile->{$pkglist}{$rpmdep} or brokendeps; $rpmdep = $group->{depslistid}[$_]; ($rpmdeplist) = find_list($config, $group, $rpmdep, $listnumber); log_("closeRpmsList: rpmdeplist $rpmdeplist\n", $config->{verbose}, $config->{LOG}, 4); } # log_("rpmdep $rpmdep rpmdeplist $rpmdeplist rpm $rpm\n",1, $config->{LOG}); if ($rpmdep) { if (!$rpmdeplist || $group->{brokendeps}{$rpmdep} == 2 || $group->{brokendeps}{$rpmdep} == 3) { $group->{brokendeps}{$rpm} = $group->{brokendeps}{$rpmdep}; push @{$group->{rejected}{$rpm}{$listnumber}}, [ "deps_rejected", $rpmdep ]; $n = 1; log_("WARNING closeRpmsList: $rpm has unresolved or excluded dependencies ($rpmdep list $rpmdeplist), removed\n", $config->{verbose}, $config->{LOG}, 1); log_("closeRpmsList: deleting $rpm (list $listnumber)\n", $config->{verbose}, $config->{LOG}, 2); delete $rpmfile->{$listnumber}{$rpm}; next } if (! defined $rpmfile->{$rpmdeplist}{$rpmdep} || ! ref $rpmfile->{$rpmdeplist}{$rpmdep}) { if (!$group->{done}{rep}{$rpmdep} || $needed && $group->{done}{rep}{$rpmdep} > $needed) { $n = 1; log_("closeRpmsList: ADDED $rpmdep (list $rpmdeplist) needed $needed (done $group->{done}{rep}{$rpmdep})\n", $config->{verbose}, $config->{LOG}, 3); # FIXME should not be needed but a bug was laying arround $rpmfile->{$rpmdeplist}{$rpmdep} ||= { }; $rpmfile->{$rpmdeplist}{$rpmdep}{needed} ||= $needed } } elsif ($needed && !$group->{done}{rep}{$rpmdep} || $group->{done}{rep}{$rpmdep} > $needed) { my $n = $rpmfile->{$rpmdeplist}{$rpmdep}{needed}; $rpmfile->{$rpmdeplist}{$rpmdep}{needed} = !$n || $needed < $n ? $needed : $n; log_("closeRpmsList: setting $rpmdep needed option to $rpmfile->{$rpmdeplist}{$rpmdep}{needed}\n", $config->{verbose}, $config->{LOG}, 6); } } if ($specialrpmdep) { if (! defined $rpmfile->{$rpmdeplist}{$specialrpmdep} || ref $rpmfile->{$listnumber}{$specialrpmdep}) { $n = 1; log_("closeRpmsList: specialrpmdep ADDED $specialrpmdep (list $listnumber)\n", $config->{verbose}, $config->{LOG}, 3); $rpmfile->{$listnumber}{$specialrpmdep} ||= { } } } } $done{$rpm} = 1; } log_("closeRpmsList: $listnumber [$n]\n", $config->{verbose}, $config->{LOG}, 2); } } } sub addRPMToList { my ($group, $listnumber, $rpmfile, $done, $rpms, $fentry, $name, $test) = @_; my $exclude = sub { my ($rpm, $name) = @_; print " Exclude $_\n" if $test; log_("addRPMToList: excluding $_\n",1, $config->{LOG}, 2); $group->{brokendeps}{$rpm} = 3; push @{$group->{rejected}{$rpm}{$listnumber}}, ["excluded", $name]; }; $name =~ s/\+/\\+/g; my @toadd; if ($fentry->{regexp}) { @toadd = grep { /$name/ } @$rpms } else { @toadd = grep { /^$name-[^-]+-[^-]+\.[^.]*$/ } @$rpms } log_("addRPMToList: toadd $name (regexp $fentry->{regexp}) (@toadd)\n", $config->{verbose}, $config->{LOG}, 3); if ($fentry->{done}) { foreach (@toadd) { my ($pkg) = /^(.*)-[^-]+-[^-]+\.[^.]*$/; my %ht; foreach my $k (keys %$fentry) { $ht{$k} = $fentry->{$k} } $rpmfile->{$listnumber}{$_} = \%ht; my $a = [ $_, $group->{size}{$_}{$listnumber}[2], \%ht, $listnumber ]; $done->{name}{$pkg} = $a; $done->{full}{$pkg} = 1; log_("addRPMToList: ADDED done $_ (list $listnumber)\n", $config->{verbose}, $config->{LOG}, 4) } return } my %pkg; if ($fentry->{regexp}) { foreach (@toadd) { if ($rpmfile->{$listnumber}{$_}) { log_("WARNING addRPMToList: do not replace $_ entry in rpmfile\n", $config->{verbose}, $config->{LOG}, 1); next } if (!$_) { log_("ERROR addRPMToList: empty rpm\n", $config->{verbose}, $config->{LOG}, 2); next } $group->{size}{$_}{$listnumber} or next; $group->{brokendeps}{$_} == 2 and next; $group->{brokendeps}{$_} == 3 and next; my ($pkgname) = /^(.*)-[^-]+-[^-]+\.[^.]*$/; $done->{full}{$_} and next; my $rep = $group->{size}{$_}{$listnumber}[2]; if ($fentry->{exclude}) { $exclude->($_, $name); next } if (!$group->{options}{dup} && $done->{name}{$pkgname} && $done->{name}{$pkgname}[3] == $listnumber) { if (!$fentry->{update} || !$done->{name}{$pkgname}[2]{done}) { if ($rep < $done->{name}{$pkgname}[1]) { $pkg{$done->{name}{$pkgname}[0]} = 0; log_("REPLACING $done->{name}{$pkgname}[0] with $_ (list $listnumber)\n", $config->{verbose}, $config->{LOG}, 4); print "Replace $done->{name}{$pkgname}[0] with $_\n" if $test; $pkg{$_} = 1; my $a = [ $_, $rep, $fentry, $listnumber ]; $done->{name}{$pkgname} = $a; $done->{full}{$_} = 1 } elsif ($done->{name}{$pkgname}[1] == $rep) { if (rpmVersionCompare($done->{name}{$pkgname}[0], $_) < 0) { $pkg{$done->{name}{$pkgname}[0]} = 0; log_("REPLACING $done->{name}{$pkgname}[0] with $_ (list $listnumber)\n", $config->{verbose}, $config->{LOG}, 4); print "Replace $done->{name}{$pkgname}[0] with $_\n" if $test; $pkg{$_} = 1; my $a = [ $_, $rep, $fentry, $listnumber ]; $done->{name}{$pkgname} = $a; $done->{full}{$_} = 1 } } } } else { $pkg{$_} = 1; my $a = [ $_, $rep, $fentry, $listnumber ]; $done->{name}{$pkgname} = $a; $done->{full}{$_} = 1 } } } else { my $rep = -1; my $pkg; # FIXME present algorythm selects only one package per version, and choose the one in the list declared first. # Maybe adding all the version and letting closeRRPMsList choose the right one is better. foreach (@toadd) { if (!$_) { log_("ERROR addRPMToList: empty rpm\n",1, $config->{LOG}, 2); next } if ($rpmfile->{$listnumber}{$_}) { log_("WARNING addRPMToList: do not replace $_ entry in rpmfile\n", $config->{verbose}, $config->{LOG}, 1); next } $group->{size}{$_}{$listnumber} or next; $group->{brokendeps}{$_} == 2 and next; $group->{brokendeps}{$_} == 3 and next; if ($fentry->{exclude}) { $exclude->($_, $name); next; } if ($group->{size}{$_}{$listnumber}[2] < $rep || $rep == -1) { $rep = $group->{size}{$_}{$listnumber}[2]; log_("addRPMToList: choosing $_ (rep $rep)\n", $config->{verbose}, $config->{LOG}, 2); $pkg = $_ } elsif ($group->{size}{$_}{$listnumber}[2] == $rep) { if (rpmVersionCompare($pkg, $_) < 0) { $rep = $group->{size}{$_}{$listnumber}[2]; log_("addRPMToList: choosing $_ (rep $rep)\n", $config->{verbose}, $config->{LOG}, 2); $pkg = $_ } } } my ($pkgname) = $pkg =~ /^(.*)-[^-]+-[^-]+\.[^.]*$/; if ($group->{options}{dup} || !$done->{name}{$pkgname}) { $pkg{$pkg} = 1; my $a = [ $pkg, $rep, $fentry, $listnumber ]; $done->{name}{$pkgname} = $a; $done->{full}{$pkg} = 1 } } $fentry->{exclude} and return 1; foreach (keys %pkg) { if ($rpmfile->{$listnumber}{$_}) { log_("WARNING addRPMToList: do not replace $_ entry in rpmfile\n", $config->{verbose}, $config->{LOG}, 1); next } $pkg{$_} or next; defined $fentry->{rate} and $group->{pkgrate}{$_} = $fentry->{rate} and log_("addRPMToList: setting $_ rate to $fentry->{rate}\n",1, $config->{LOG}, 2); my %ht; foreach my $k (keys %$fentry) { $ht{$k} = $fentry->{$k} } $rpmfile->{$listnumber}{$_} = \%ht; print " $_\n" if $test; log_("addRPMToList: ADDED $_ (list $listnumber) options " . join(" ", keys %ht) . "\n", $config->{verbose}, $config->{LOG}, 4) } } sub build_list { my ($class, $group, $test) = @_; my %rpmfile; my $filelist = $group->{filelist}; my @fullrpm = keys %{$group->{urpm}{rpm}}; my @section = keys %{$group->{rpmsrate}[1]}; my %done; foreach my $listnumber (@{$group->{orderedlist}{rpm}}) { log_("build_list: list $listnumber ($group->{listrpm}{$listnumber})\n", $config->{verbose}, $config->{LOG}, 2); my $rpms = $group->{listrpm}{$listnumber}; my $needed; next if $config->{list}[$listnumber]{done} && !$test; if ($config->{list}[$listnumber]{pseudo_done}) { # Take the first one, anyway the pseudo_done attribute must only appear for single directory list (fixed command) my ($cd, $rep) = @{$group->{list}{$listnumber}{rpm}[0]}; $needed = $group->{orderedrep}{rpm}{"$cd/$rep"} } if (ref $rpms) { log_("$listnumber -- $group->{filelist} -- " . keys(%{$group->{filelist}}) . "\n", $config->{verbose}, $config->{LOG}, 3); if (!ref $filelist->{$listnumber}) { log_("WARNING: list $listnumber has an empty file list\n", $config->{verbose}, $config->{LOG}, 1); next } log_("build_list: FILE LIST $listnumber (" . int @{$filelist->{$listnumber}} . ")\n", $config->{verbose}, $config->{LOG}, 4); foreach my $fentry (@{$filelist->{$listnumber}}) { my $name = $fentry->[0]; my $opt = $fentry->[1]; log_("build_list: processing $name " . join(' ', keys %$opt) . "\n", $config->{verbose}, $config->{LOG}, 5); print "\nChecking $name ", join(' ', keys %$opt), "\n" if $test; my @toadd; if (! defined $opt->{needed} && $needed) { $opt->{needed} = $needed } if ($opt->{section}) { my $level = $opt->{section}; log_("build_list: selecting rpmsrate package of section $name with score higher than $level\n", $config->{verbose}, $config->{LOG}, 2); $opt->{section} = 0; if ($opt->{regexp}) { $opt->{regexp} = 0; @toadd = grep { /$name/ } @section; log_("$name (@section) -> @toadd\n", $config->{verbose}, $config->{LOG}, 4); foreach (@toadd) { foreach (@{$group->{rpmsrate}[1]{$_}}) { log_("$_ -> $group->{rpmsrate}[0]{$_}\n", $config->{verbose}, $config->{LOG}, 4); if ($group->{rpmsrate}[0]{$_} >= $level) { addRPMToList($group, $listnumber, \%rpmfile, \%done, $rpms, $opt, $_, $test); } } } } else { my $rpmlist = $group->{rpmsrate}[1]{$name} or do { log_("ERROR build_list: $name unknown rpmsrate section\n", $config->{verbose}, $config->{LOG}, 0); next }; foreach (@$rpmlist) { if ($group->{rpmsrate}[0]{$_} >= $level) { addRPMToList($group, $listnumber, \%rpmfile, \%done, $rpms, $opt, $_, $test) } } } } else { addRPMToList($group, $listnumber, \%rpmfile, \%done, $rpms, $opt, $name, $test); } } } else { log_("WARNING: List $listnumber is empty, ignoring\n", $config->{verbose}, $config->{LOG}, 0); $class->{config}{list}[$listnumber]{empty} = 1; } } $test and return; my $revdeps; if (!$class->{config}{nodeps} && !$group->{options}{nodeps}) { my @toadd = grep { /^basesystem-[^-]+-[^-]+\.[^.]*$/ } @fullrpm; my $pkg; my $listnumber; foreach (@toadd) { my ($l) = find_list($config,$group, $_); if ($group->{listmatrix}{rpm}{$listnumber}{$l} || !$listnumber) { $pkg = $_; $listnumber = $l } } if ($pkg) { $rpmfile{$listnumber}{$pkg} = {}; log_("build_list ADDED $pkg (list $listnumber)\n", $config->{verbose}, $config->{LOG}, 4); } else { log_("ERROR: basesystem package is not available.\n",1, $config->{LOG}) } foreach my $pkg (keys %{$rpmfile{1}}) { log_("build_list CURRENT 1 $pkg\n", $config->{verbose}, $config->{LOG}, 4); } $revdeps = reverseDepslist($class, $group, \%rpmfile); foreach my $pkg (keys %{$rpmfile{1}}) { log_("build_list CURRENT 4 $pkg\n", $config->{verbose}, $config->{LOG}, 4); } # add deps closeRpmsList($group, \%rpmfile); foreach my $pkg (keys %{$rpmfile{1}}) { log_("build_list CURRENT 2 $pkg\n", $config->{verbose}, $config->{LOG}, 4); } } (\%rpmfile, $revdeps) } 1 # Changelog # # 2002 02 21 # change false $j comparaison to $depsdisc in buildDisc to new $thisorderrep value. # # 2002 03 03 # new limit option handling. # add updateGenericSoft function # add testSoftLimit function # update size to check rep size # # 2002 03 08 # fix autoMode CD adding # # 2002 03 13 # better selection of alternatives in multi-list to take the one in the first lists. # # 2002 03 14 # add sources new sources handling method # in nosrcfit mode sources are added afterwards # # 2002 03 19 # add prelist in geList for cdcom, will be useful for oem too I guess. # # 2002 05 02 # add_one_disc: add separate mode for sources mode # # 2002 05 09 # add graft structure for md5 and graft point handling # # 2002 05 13 # fix a tricky bugs in build_list about fentry shared and not recreated for each packages. # # 2002 06 01 # use perl-URPM # # 2002 06 15 # new diff mode, global, shared between disc and group, only one table. # # 2002 08 16 # new diff_idx table to sort diff data # # 2002 08 24 # optimize_space first version, still need to handle correctly needed and more advanced optimization methods. # # 2002 09 18 # optimize_space work, fixes and updates. # # 2002 10 25 # fix needed assignation pb in closeRPMslist # # 2004 05 27 # separate List.pm into List.pm and Build.pm