From padb at googlecode.com Sat Jun 18 22:49:20 2011 From: padb at googlecode.com (padb at googlecode.com) Date: Sat, 18 Jun 2011 21:49:20 +0000 Subject: [padb] r439 committed - Re-indent after the last commit. Message-ID: Revision: 439 Author: apittman at gmail.com Date: Sat Jun 18 14:48:57 2011 Log: Re-indent after the last commit. http://code.google.com/p/padb/source/detail?r=439 Modified: /trunk/src/padb ======================================= --- /trunk/src/padb Fri May 20 13:01:28 2011 +++ /trunk/src/padb Sat Jun 18 14:48:57 2011 @@ -9219,7 +9219,9 @@ my @frames = @{ $thread->{frames} }; foreach my $i ( reverse 0 .. $#frames ) { my $frame = $frames[$i]; - if ( defined $frame->{func} and defined $fns->{fns}{ $frame->{func} } ) { + if ( defined $frame->{func} + and defined $fns->{fns}{ $frame->{func} } ) + { $fnmode = $fns->{fns}{ $frame->{func} }; last; } From padb at googlecode.com Sat Jun 18 23:21:32 2011 From: padb at googlecode.com (padb at googlecode.com) Date: Sat, 18 Jun 2011 22:21:32 +0000 Subject: [padb] r440 committed - Tweak the "tree" view output slightly, add a column at the left which ... Message-ID: <20cf300fb15db0564004a603ece7@google.com> Revision: 440 Author: apittman at gmail.com Date: Sat Jun 18 15:21:15 2011 Log: Tweak the "tree" view output slightly, add a column at the left which shows shows the current indentation level so it's easier to match different branches of the tree. As it's not possible to match branches by the number only indent each subsequent level by 1 rather than 2 characters to make the tree narrower and (hopefully) easier to read. http://code.google.com/p/padb/source/detail?r=440 Modified: /trunk/src/padb ======================================= --- /trunk/src/padb Sat Jun 18 14:48:57 2011 +++ /trunk/src/padb Sat Jun 18 15:21:15 2011 @@ -4377,8 +4377,9 @@ } sub _display_tree { - my ( $tree, $parent, $indent, $path, $enforce_spec ) = @_; - + my ( $tree, $parent, $indent_count, $path, $enforce_spec ) = @_; + + my $indent = sprintf( "%3d: %*s", $indent_count, $indent_count, '' ); my $ret = $EMPTY_STRING; # Sort peers by lowest rank of each branch. @@ -4405,8 +4406,11 @@ } if ( defined $tree->{$peer}->{desc} ) { - $ret .= _display_tree( $tree->{$peer}->{desc}, - $vpspec, "$indent ", "$path,$peer", $child_enforce_spec ); + $ret .= _display_tree( + $tree->{$peer}->{desc}, $vpspec, + $indent_count + 1, "$path,$peer", + $child_enforce_spec + ); } } return $ret; @@ -4414,7 +4418,7 @@ sub display_tree { my ( $tree, ) = @_; - return _display_tree( $tree, "no-parent", $EMPTY_STRING, $EMPTY_STRING, 1 ); + return _display_tree( $tree, "no-parent", 0, $EMPTY_STRING, 1 ); } # An experimental new tree format. From padb at googlecode.com Sun Jun 19 00:24:46 2011 From: padb at googlecode.com (padb at googlecode.com) Date: Sat, 18 Jun 2011 23:24:46 +0000 Subject: [padb] r441 committed - Another change to the treeview display, for every branch in the tree... Message-ID: <20cf300fb15dcefc6304a604ce71@google.com> Revision: 441 Author: apittman at gmail.com Date: Sat Jun 18 16:24:16 2011 Log: Another change to the treeview display, for every branch in the tree not only display the rankspec and the number of processes but also the percentage of global processes this is. http://code.google.com/p/padb/source/detail?r=441 Modified: /trunk/src/padb ======================================= --- /trunk/src/padb Sat Jun 18 15:21:15 2011 +++ /trunk/src/padb Sat Jun 18 16:24:16 2011 @@ -4377,7 +4377,7 @@ } sub _display_tree { - my ( $tree, $parent, $indent_count, $path, $enforce_spec ) = @_; + my ( $tree, $nproc, $parent, $indent_count, $path, $enforce_spec ) = @_; my $indent = sprintf( "%3d: %*s", $indent_count, $indent_count, '' ); my $ret = $EMPTY_STRING; @@ -4392,7 +4392,14 @@ my $vpspec = rng_convert_to_user( $tree->{$peer}->{range} ); if ( @peers != 1 or $parent ne $vpspec or $enforce_spec ) { $ret .= "$indent-----------------\n"; - $ret .= "$indent$vpspec ($tree->{$peer}->{count} processes)\n"; + my $count = $tree->{$peer}->{count}; + + $ret .= sprintf( + "$indent$vpspec ($count %s %.3g%%)\n", + $count > 1 ? 'processes' : 'process', + ( $count / $nproc ) * 100 + ); + $ret .= "$indent-----------------\n"; } @@ -4407,9 +4414,9 @@ if ( defined $tree->{$peer}->{desc} ) { $ret .= _display_tree( - $tree->{$peer}->{desc}, $vpspec, - $indent_count + 1, "$path,$peer", - $child_enforce_spec + $tree->{$peer}->{desc}, + $nproc, $vpspec, $indent_count + 1, + "$path,$peer", $child_enforce_spec ); } } @@ -4417,14 +4424,16 @@ } sub display_tree { - my ( $tree, ) = @_; - return _display_tree( $tree, "no-parent", 0, $EMPTY_STRING, 1 ); + my ( $tree, $nproc ) = @_; + return _display_tree( $tree, $nproc, "no-parent", 0, $EMPTY_STRING, 1 ); } # An experimental new tree format. sub new_tree { my ( $lines, $d ) = @_; my %tree; + + my $nprocesses = keys %{$lines}; debug_log( 'tree', $d, 'Making the tree' ); foreach my $tag ( sort { $a <=> $b } keys %{$lines} ) { add_tag_to_tree( \%tree, $tag, $lines->{$tag} ); @@ -4432,7 +4441,7 @@ debug_log( 'tree', \%tree, 'Enhancing the tree' ); add_data_to_tree( \%tree, undef, $d ); debug_log( 'tree', \%tree, 'Formatting the tree' ); - my $t = display_tree( \%tree, ); + my $t = display_tree( \%tree, $nprocesses ); debug_log( 'tree', undef, 'Displaying the tree' ); print $t; debug_log( 'tree', undef, 'Done' ); @@ -4444,6 +4453,7 @@ my ( $d, $ns ) = @_; my %tree; + my $nprocesses = keys %{ $d->{target_ns_output} }; debug_log( 'tree', undef, 'Making the tree' ); my @tags; foreach my $tag ( keys %{ $d->{target_ns_output} } ) { @@ -4457,7 +4467,7 @@ debug_log( 'tree', \%tree, 'Enhancing the tree' ); add_data_to_tree( \%tree, $ns, $d ); debug_log( 'tree', \%tree, 'Formatting the tree' ); - my $t = display_tree( \%tree, ); + my $t = display_tree( \%tree, $nprocesses ); debug_log( 'tree', undef, 'Displaying the tree' ); print "Stack trace(s) for thread: $ns\n"; print $t; From padb at googlecode.com Thu Jun 23 00:18:54 2011 From: padb at googlecode.com (padb at googlecode.com) Date: Wed, 22 Jun 2011 23:18:54 +0000 Subject: [padb] r442 committed - In tree mode add the option to truncate the "rankspec" list of stack f... Message-ID: <20cf300fb0f936863504a6553149@google.com> Revision: 442 Author: apittman at gmail.com Date: Wed Jun 22 16:17:55 2011 Log: In tree mode add the option to truncate the "rankspec" list of stack frames. This causes padb to display less information but avoids the problem on large jobs where the list of ranks can be several lines long at times. Truncate the rankspec to be 'N' entries long where each entry is a comma seperated part, that is '1' or '12-17' would be example entries. Limit the number of entries by only showing the first ones in the list and adding ellipses if not all are shown. This is going to need to be made tunable and a sensible default chosen. http://code.google.com/p/padb/source/detail?r=442 Modified: /trunk/src/padb ======================================= --- /trunk/src/padb Sat Jun 18 16:24:16 2011 +++ /trunk/src/padb Wed Jun 22 16:17:55 2011 @@ -4285,8 +4285,10 @@ my @values = keys %{ $d->{target_data}{$key} }; if ( @values == 1 ) { - my $line = " $vref->{type_name} = '$values[0]' " - . rng_convert_to_user( $d->{target_data}{$key}{ $values[0] } ); + my $line = + " $vref->{type_name} = '$values[0]' " + . rng_convert_to_user_truncate( + $d->{target_data}{$key}{ $values[0] } ); push @{ $tree->{$peer}->{aux} }, $line; } elsif ( @values > $max_show ) { my $line = " $vref->{type_name}: " @@ -4295,8 +4297,10 @@ } else { push @{ $tree->{$peer}->{aux} }, " $vref->{type_name}:"; foreach my $value ( sort @values ) { - my $line = " '$value' " - . rng_convert_to_user( $d->{target_data}{$key}{$value} ); + my $line = + " '$value' " + . rng_convert_to_user_truncate( + $d->{target_data}{$key}{$value} ); push @{ $tree->{$peer}->{aux} }, $line; } @@ -4389,7 +4393,7 @@ my $child_enforce_spec = 0; foreach my $peer (@peers) { - my $vpspec = rng_convert_to_user( $tree->{$peer}->{range} ); + my $vpspec = rng_convert_to_user_truncate( $tree->{$peer}->{range} ); if ( @peers != 1 or $parent ne $vpspec or $enforce_spec ) { $ret .= "$indent-----------------\n"; my $count = $tree->{$peer}->{count}; @@ -5214,6 +5218,19 @@ map { $_->{l} == $_->{u} ? $_->{l} : $_->{l} . q{-} . $_->{u} } @{$rg}; return "[$range]"; } + +sub rng_convert_to_user_truncate { + my ($rg) = @_; + + my $max_rankspec_length = 5; + my @spec = + map { $_->{l} == $_->{u} ? $_->{l} : $_->{l} . q{-} . $_->{u} } @{$rg}; + if ( $#spec >= $max_rankspec_length ) { + return sprintf( "[%s...]", + join q{,}, @spec[ 0 .. $max_rankspec_length - 1 ] ); + } + return sprintf( "[%s]", join q{,}, @spec ); +} sub rng_shift { my ($rg) = @_; From padb at googlecode.com Fri Jun 24 00:15:35 2011 From: padb at googlecode.com (padb at googlecode.com) Date: Thu, 23 Jun 2011 23:15:35 +0000 Subject: [padb] r443 committed - Add a check that the PID given as the process to debug isnt padb itsel... Message-ID: <20cf300fb3f5235fae04a6694336@google.com> Revision: 443 Author: apittman at gmail.com Date: Thu Jun 23 16:14:50 2011 Log: Add a check that the PID given as the process to debug isnt padb itself! previously if this happened padb would hang, not it errors with a somewhat helpful warning. http://code.google.com/p/padb/source/detail?r=443 Modified: /trunk/src/padb ======================================= --- /trunk/src/padb Wed Jun 22 16:17:55 2011 +++ /trunk/src/padb Thu Jun 23 16:14:50 2011 @@ -9285,6 +9285,9 @@ sub register_target_process { my ( $rank, $pid ) = @_; + if ( $pid == $$ ) { + croak("Error locating processes, refusing to debug self"); + } $inner_conf{rmpids}{$pid}{rank} = $rank; return; } @@ -10325,7 +10328,7 @@ my $count = sysread $s, $d, 65536; # Dead connection. - if ( not defined $d or $count == 0 ) { + if ( not defined $d or not defined $count or $count == 0 ) { if ( eof $s ) { $sel->remove($s); From padb at googlecode.com Fri Jun 24 00:19:36 2011 From: padb at googlecode.com (padb at googlecode.com) Date: Thu, 23 Jun 2011 23:19:36 +0000 Subject: [padb] r444 committed - Protect agains the case where padb is run against slurm and is given a... Message-ID: <000e0cd111768c2e6c04a6695167@google.com> Revision: 444 Author: apittman at gmail.com Date: Thu Jun 23 16:17:17 2011 Log: Protect agains the case where padb is run against slurm and is given a allocation id for which no job steps exist. In this case padb exit saying it failed to locate and ranks. http://code.google.com/p/padb/source/detail?r=444 Modified: /trunk/src/padb ======================================= --- /trunk/src/padb Thu Jun 23 16:14:50 2011 +++ /trunk/src/padb Thu Jun 23 16:17:17 2011 @@ -9359,6 +9359,7 @@ foreach my $proc (@procs) { my ( $pid, $job, $step, undef, $global ) = split $SPACE, $proc; next if ( $global eq '-' ); + next if ( $pid == $$ ); next unless ( $job eq $jobid ); next unless ( $step == $inner_conf{slurm_job_step} ); next if ( is_resmgr_process($pid) ); From padb at googlecode.com Fri Jun 24 20:05:00 2011 From: padb at googlecode.com (padb at googlecode.com) Date: Fri, 24 Jun 2011 19:05:00 +0000 Subject: [padb] r445 committed - Change the order of the tests in the last commit to avoid... Message-ID: <20cf302ef790d3114a04a679e059@google.com> Revision: 445 Author: apittman at gmail.com Date: Fri Jun 24 12:04:08 2011 Log: Change the order of the tests in the last commit to avoid a warning message about non numeric compares. http://code.google.com/p/padb/source/detail?r=445 Modified: /trunk/src/padb ======================================= --- /trunk/src/padb Thu Jun 23 16:17:17 2011 +++ /trunk/src/padb Fri Jun 24 12:04:08 2011 @@ -9359,8 +9359,8 @@ foreach my $proc (@procs) { my ( $pid, $job, $step, undef, $global ) = split $SPACE, $proc; next if ( $global eq '-' ); - next if ( $pid == $$ ); next unless ( $job eq $jobid ); + next if ( $pid == $$ ); next unless ( $step == $inner_conf{slurm_job_step} ); next if ( is_resmgr_process($pid) ); register_target_process( $global, $pid ); From padb at googlecode.com Tue Jun 28 22:42:00 2011 From: padb at googlecode.com (padb at googlecode.com) Date: Tue, 28 Jun 2011 21:42:00 +0000 Subject: [padb] r446 committed - Another change to the tree based display, move the rankspec to the... Message-ID: <00151750e880b5bbe604a6cc8991@google.com> Revision: 446 Author: apittman at gmail.com Date: Tue Jun 28 14:41:38 2011 Log: Another change to the tree based display, move the rankspec to the end of the line, after the number of processes and other data. Stack traces now look similar to below. Stack trace(s) for thread: 1 1: ----------------- 1: Branch 1 of 1, 17 processes, 100% of total [0-16] 1: ----------------- 1: main() at IMB.c:228 2: ----------------- 2: Branch 1 of 2, 2 processes, 11.8% of total [0-1] 2: ----------------- 2: IMB_pingping() at IMB_pingping.c:148 3: PMPI_Recv() at /home/ashley/OpenMPI/openmpi-1.4.3/ompi/mpi/c/profile/precv.c:75 4: mca_pml_ob1_recv() at /home/ashley/OpenMPI/openmpi-1.4.3/ompi/mca/pml/ob1/pml_ob1_irecv.c:104 5: ompi_request_wait_completion() at /home/ashley/OpenMPI/openmpi-1.4.3/ompi/request/request.h:375 6: opal_condition_wait() at /home/ashley/OpenMPI/openmpi-1.4.3/opal/threads/condition.h:99 2: ----------------- 2: Branch 2 of 2, 15 processes, 88.2% of total [2-16] 2: ----------------- 2: PMPI_Barrier() at /home/ashley/OpenMPI/openmpi-1.4.3/ompi/mpi/c/profile/pbarrier.c:59 3: ompi_coll_tuned_barrier_intra_dec_fixed() at /home/ashley/OpenMPI/openmpi-1.4.3/ompi/mca/coll/tuned/coll_tuned_decision_fixed.c:203 4: ompi_coll_tuned_barrier_intra_bruck() at /home/ashley/OpenMPI/openmpi-1.4.3/ompi/mca/coll/tuned/coll_tuned_barrier.c:227 5: ompi_coll_tuned_sendrecv_actual() at /home/ashley/OpenMPI/openmpi-1.4.3/ompi/mca/coll/tuned/coll_tuned_util.c:55 6: ompi_request_default_wait_all() at /home/ashley/OpenMPI/openmpi-1.4.3/ompi/request/req_wait.c:262 7: opal_condition_wait() at /home/ashley/OpenMPI/openmpi-1.4.3/opal/threads/condition.h:99 http://code.google.com/p/padb/source/detail?r=446 Modified: /trunk/src/padb ======================================= --- /trunk/src/padb Fri Jun 24 12:04:08 2011 +++ /trunk/src/padb Tue Jun 28 14:41:38 2011 @@ -4383,7 +4383,7 @@ sub _display_tree { my ( $tree, $nproc, $parent, $indent_count, $path, $enforce_spec ) = @_; - my $indent = sprintf( "%3d: %*s", $indent_count, $indent_count, '' ); + my $indent = sprintf( "%3d:%*s", $indent_count + 1, $indent_count, '' ); my $ret = $EMPTY_STRING; # Sort peers by lowest rank of each branch. @@ -4391,28 +4391,31 @@ sort { $tree->{$a}->{min} <=> $tree->{$b}->{min} } keys %{$tree}; my $child_enforce_spec = 0; + my $branch_index = 1; foreach my $peer (@peers) { my $vpspec = rng_convert_to_user_truncate( $tree->{$peer}->{range} ); if ( @peers != 1 or $parent ne $vpspec or $enforce_spec ) { - $ret .= "$indent-----------------\n"; + $ret .= "$indent -----------------\n"; my $count = $tree->{$peer}->{count}; $ret .= sprintf( - "$indent$vpspec ($count %s %.3g%%)\n", + "$indent Branch %d of %d, $count %s, %.3g%% of total $vpspec\n", + $branch_index, + $#peers + 1, $count > 1 ? 'processes' : 'process', ( $count / $nproc ) * 100 ); - $ret .= "$indent-----------------\n"; + $ret .= "$indent -----------------\n"; } - $ret .= "$indent$peer\n"; + $ret .= "$indent $peer\n"; if ( defined $tree->{$peer}->{aux} ) { $child_enforce_spec = 1; foreach my $line ( @{ $tree->{$peer}->{aux} } ) { - $ret .= "$indent $line\n"; + $ret .= "$indent $line\n"; } } @@ -4423,6 +4426,7 @@ "$path,$peer", $child_enforce_spec ); } + $branch_index += 1; } return $ret; }