1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
|
#+OPTIONS: H:3 num:nil toc:t
#+TITLE: org-babel --- facilitating communication between programming languages and people
#+SEQ_TODO: TODO PROPOSED | DONE DEFERRED REJECTED
#+STARTUP: oddeven hideblocks
* Introduction
Org-Babel enables *communication* between programming languages and
between people.
Org-Babel provides:
- communication between programs :: Data passes seamlessly between
different programming languages, text, and tables.
- communication between people :: Data and calculations are embedded
in the same document as notes explanations and reports.
** communication between programs
Org-Mode supports embedded blocks of source code (in any language)
inside of Org documents. Org-Babel allows these blocks of code to be
executed from within Org-Mode with natural handling of their inputs
and outputs.
*** simple execution
with both scalar, file, and table output
*** reading information from tables
*** reading information from other source blocks (disk usage in your home directory)
This will work for Linux and Mac users, not so sure about shell
commands for windows users.
To run place the cursor on the =#+begin_src= line of the source block
labeled directory-pie and press =\C-c\C-c=.
#+srcname: directories
#+begin_src bash :results replace
cd ~ && du -sc * |grep -v total
#+end_src
#+resname: directories
| 64 | "Desktop" |
| 11882808 | "Documents" |
| 8210024 | "Downloads" |
| 879800 | "Library" |
| 57344 | "Movies" |
| 7590248 | "Music" |
| 5307664 | "Pictures" |
| 0 | "Public" |
| 152 | "Sites" |
| 8 | "System" |
| 56 | "bin" |
| 3274848 | "mail" |
| 5282032 | "src" |
| 1264 | "tools" |
#+srcname: directory-pie
#+begin_src R :var dirs = directories
pie(dirs[,1], labels = dirs[,2])
#+end_src
*** operations in/on tables
#+tblname: grades-table
| student | grade | letter |
|---------+-------+--------|
| 1 | 99 | A |
| 2 | 59 | F |
| 3 | 75 | C |
| 4 | 15 | F |
| 5 | 7 | F |
| 6 | 13 | F |
#+TBLFM: $2='(sbe random-score-generator)::$3='(sbe assign-grade (score $2))
#+srcname: assign-grade
#+begin_src ruby :var score=99
case score
when 0..59: "F"
when 60..69: "D"
when 70..79: "C"
when 80..89: "B"
when 90..100: "A"
else "Invalid Score"
end
#+end_src
#+srcname: random-score-generator
#+begin_src ruby
rand(100)
#+end_src
#+srcname: show-distribution
#+begin_src R :var grades=grades-table
hist(grades[,2])
#+end_src
** communication between people
Quick overview of Org-Mode's exportation abilities, with links to the
online Org-Mode documentation, a focus on source-code blocks, and the
exportation options provided by Org-Babel.
*** Interactive tutorial
This would demonstrate applicability to Reproducible Research, and
Literate Programming.
*** Tests embedded in documentation
org-babels own functional tests are contained in a large org-mode
table, allowing the test suite to be run be evaluation of the table
and the results to be collected in the same table.
* Tasks [22/36]
** TODO Create objects in top level (global) environment [0/5]
*sessions*
*** initial requirement statement [DED]
At the moment, objects created by computations performed in the
code block are evaluated in the scope of the
code-block-function-body and therefore disappear when the code
block is evaluated {unless you employ some extra trickery like
assign('name', object, env=globalenv()) }. I think it will be
desirable to also allow for a style wherein objects that are
created in one code block persist in the R global environment and
can be re-used in a separate block.
This is what Sweave does, and while I'm not saying we have to be
the same as Sweave, it wouldn't be hard for us to provide the same
behaviour in this case; if we don't, we risk undeservedly being
written off as an oddity by some.
IOW one aspect of org-babel is that of a sort of functional
meta-programming language. This is crazy, in a very good
way. Nevertheless, wrt R I think there's going to be a lot of value
in providing for a working style in which the objects are stored in
the R session, rather than elisp/org buffer. This will be a very
familiar working style to lots of people.
There are no doubt a number of different ways of accomplishing
this, the simplest being a hack like adding
#+begin_src R
for(objname in ls())
assign(objname, get(objname), envir=globalenv())
#+end_src
to the source code block function body. (Maybe wrap it in an on.exit() call).
However this may deserve to be thought about more carefully, perhaps
with a view to having a uniform approach across languages. E.g. shell
code blocks have the same semantics at the moment (no persistence of
variables across code blocks), because the body is evaluated in a new
bash shell process rather than a running shell. And I guess the same
is true for python. However, in both these cases, you could imagine
implementing the alternative in which the body is evaluated in a
persistent interactive session. It's just that it's particularly
natural for R, seeing as both ESS and org-babel evaluate commands in a
single persistent R session.
*** sessions [Eric]
Thanks for bringing this up. I think you are absolutely correct that we
should provide support for a persistent environment (maybe called a
*session*) in which to evaluate code blocks. I think the current setup
demonstrates my personal bias for a functional style of programming
which is certainly not ideal in all contexts.
While the R function you mention does look like an elegant solution, I
think we should choose an implementation that would be the same across
all source code types. Specifically I think we should allow the user to
specify an optional *session* as a header variable (when not present we
assume a default session for each language). The session name could be
used to name a comint buffer (like the *R* buffer) in which all
evaluation would take place (within which variables would retain their
values --at least once I remove some of the functional method wrappings
currently in place-- ).
This would allow multiple environments to be used in the same buffer,
and once this setup was implemented we should be able to fairly easily
implement commands for jumping between source code blocks and the
related session buffers, as well as for dumping the last N commands from
a session into a new or existing source code block.
Please let me know if you foresee any problems with this proposed setup,
or if you think any parts might be confusing for people coming from
Sweave. I'll hopefully find some time to work on this later in the
week.
*** can functional and interpreted/interactive models coexist?
Even though both of these use the same =*R*= buffer the value of =a=
is not preserved because it is assigned inside of a functional
wrapper.
#+srcname: task-R-sessions
#+begin_src R
a <- 9
b <- 21
a + b
#+end_src
#+srcname: task-R-same-session
#+begin_src R
a
#+end_src
This functional wrapper was implemented in order to efficiently return
the results of the execution of the entire source code block. However
it inhibits the evaluation of source code blocks in the top level,
which would allow for persistence of variable assignment across
evaluations. How can we allow *both* evaluation in the top level, and
efficient capture of the return value of an entire source code block
in a language independent manner?
Possible solutions...
1) we can't so we will have to implement two types of evaluation
depending on which is appropriate (functional or imperative)
2) we remove the functional wrapper and parse the source code block
into it's top level statements (most often but not always on line
breaks) so that we can isolate the final segment which is our
return value.
3) we add some sort of "#+return" line to the code block
4) we take advantage of each languages support for meta-programming
through =eval= type functions, and use said to evaluate the entire
blocks in such a way that their environment can be combined with the
global environment, and their results are still captured.
5) I believe that most modern languages which support interactive
sessions have support for a =last_result= type function, which
returns the result of the last input without re-calculation. If
widely enough present this would be the ideal solution to a
combination of functional and imperative styles.
None of these solutions seem very desirable, but for now I don't see
what else would be possible.
Of these options I was leaning towards (1) and (4) but now believe
that if it is possible option (5) will be ideal.
**** (1) both functional and imperative evaluation
Pros
- can take advantage of built in functions for sending regions to the
inferior process
- retains the proven tested and working functional wrappers
Cons
- introduces the complication of keeping track of which type of
evaluation is best suited to a particular context
- the current functional wrappers may require some changes in order to
include the existing global context
**** (4) exploit language meta-programming constructs to explicitly evaluate code
Pros
- only one type of evaluation
Cons
- some languages may not have sufficient meta-programming constructs
**** (5) exploit some =last_value= functionality if present
Need to ensure that most languages have such a function, those without
will simply have to implement their own similar solution...
| language | =last_value= function |
|------------+-----------------------------|
| R | .Last.value |
| ruby | _ |
| python | _ |
| shell | see [[* last command for shells][last command for shells]] |
| emacs-lisp | see [[* emacs-lisp will be a special case][special-case]] |
#+srcname: task-last-value
#+begin_src ruby
82 + 18
#+end_src
***** last command for shells
Do this using the =tee= shell command, and continually pipe the output
to a file.
Got this idea from the following [[http://linux.derkeiler.com/Mailing-Lists/Fedora/2004-01/0898.html][email-thread]].
suggested from mailing list
#+srcname: bash-save-last-output-to-file
#+begin_src sh
while read line
do
bash -c "$line" | tee /tmp/last.out1
mv /tmp/last.out1 /tmp/last.out
done
#+end_src
another proposed solution from the above thread
#+srcname: bash-save-in-variable
#+begin_src sh
#!/bin/bash
# so - Save Output. Saves output of command in OUT shell variable.
OUT=`$*`
echo $OUT
#+end_src
and another
#+begin_quote
.inputrc:
"^[k": accept-line
"^M": " | tee /tmp/h_lastcmd.out ^[k"
.bash_profile:
export __=/tmp/h_lastcmd.out
If you try it, Alt-k will stand for the old Enter; use "command $__" to
access the last output.
Best,
--
Herculano de Lima Einloft Neto
#+end_quote
***** emacs-lisp will be a special case
While it is possible for emacs-lisp to be run in a console type
environment (see the =elim= function) it is *not* possible to run
emacs-lisp in a different *session*. Meaning any variable set top
level of the console environment will be set *everywhere* inside
emacs. For this reason I think that it doesn't make any sense to
worry about session support for emacs-lisp.
*** TODO rework all source codes to use inferior-processes-buffers
this will involve...
1) creating a a-list of default *session* buffers for each source language
2) functions for dumping code to the *session* buffers which can be
overridden by each source code language
3) functions for retrieving results from the *sessions* buffers which
can be overridden by each source code
*** TODO implement a *session* header argument
use this header argument to override the default *session* buffer
*** TODO remove source bodies from their functional wrappers
The current functional wrappers should be removed in favor of
incremental evaluation in inferior-source-buffers
*** TODO function to bring up inferior-process buffer
This should be callable from inside of a source-code block in an
org-mode buffer. It should evaluate the header arguments, then bring
up the inf-proc buffer using =pop-to-buffer=.
*** TODO function to dump last N lines from inf-proc buffer into the current source block
Callable with a prefix argument to specify how many lines should be
dumped into the source-code buffer.
** TODO support for working with =*Org Edit Src Example*= buffers [1/4]
*** TODO set buffer-local-process variables appropriately [DED]
I think something like this would be great. You've probably
already thought of this, but just to note it down: it would be really
nice if org-babel's notion of a buffer's 'session/process' played
nicely with ESS's notion of the buffer's session/process. ESS keeps
the current process name for a buffer in a buffer-local variable
ess-local-process-name. So one thing we will probably want to do is
make sure that the *Org Edit Src Example* buffer sets that variable
appropriately. [DED]
I had not thought of that, but I agree whole heartedly. [Eric]
Once this is done every variable should be able to dump regions into
their inferior-process buffer using major-mode functions.
*** TODO some possible requests/proposed changes for Carsten [2/3]
While I remember, some possible requests/proposed changes for Carsten
come to mind in that regard:
**** DONE Remap C-x C-s to save the source to the org buffer?
I've done this personally and I find it essential. I'm using
#+begin_src emacs-lisp
(defun org-edit-src-save ()
"Update the parent org buffer with the edited source code, save
the parent org-buffer, and return to the source code edit
buffer."
(interactive)
(let ((p (point)))
(org-edit-src-exit)
(save-buffer)
(org-edit-src-code)
(goto-char p)))
(define-key org-exit-edit-mode-map "\C-x\C-s" 'org-edit-src-save)
#+end_src
which seems to work.
I think this is great, but I think it should be implemented in the
org-mode core
**** TODO Rename buffer and minor mode?
Something shorter than *Org Edit Src Example* for the buffer
name. org-babel is bringing org's source code interaction to a
level of maturity where the 'example' is no longer
appropriate. And if further keybindings are going to be added to
the minor mode then maybe org-edit-src-mode is a better name than
org-exit-edit-mode.
Maybe we should name the buffer with a combination of the source
code and the session. I think that makes sense.
[ES] Are you also suggesting a new org-edit-src minor mode?
[DED] org-exit-edit-mode is a minor mode that already exists:
Minor mode installing a single key binding, "C-c '" to exit special edit.
org-edit-src-save now has a binding in that mode, so I guess all
I'm saying at this stage is that it's a bit of a misnomer. But
perhaps we will also have more functionality to add to that minor
mode, making it even more of a misnomer. Perhaps something like
org-src-mode would be better.
**** DEFERRED a hook called when the src edit buffer is created
This should be implemented in the org-mode core
*** DEFERRED send code to inferior process
Another thought on this topic: I think we will want users to send
chunks of code to the interpreter from within the *Org Edit Src*
buffer, and I think that's what you have in mind already. In ESS that
is done using the ess-eval-* functions. [DED]
I think we can leave this up to the major-mode in the source code
buffer, as almost every source-code major mode will have functions for
doing things like sending regions to the inferior process. If
anything we might need to set the value of the buffer local inferior
process variable. [Eric]
*** TODO optionally evaluate header references when we switch to =*Org Edit Src*= buffer
That seems to imply that the header references need to be evaluated
and transformed into the target language object when we hit C-c ' to
enter the *Org Edit Src* buffer [DED]
Good point, I heartily agree that this should be supported [Eric]
(or at least before the first time we attempt to evaluate code in that
buffer -- I suppose there might be an argument for lazy evaluation, in
case someone hits C-c ' but is "just looking" and not actually
evaluating anything.) Of course if evaluating the reference is
computationally intensive then the user might have to wait before they
get the *Org Edit Src* buffer. [DED]
I fear that it may be hard to anticipate when the references will be
needed, some major-modes do on-the-fly evaluation while the buffer is
being edited. I think that we should either do this before the buffer
is opened or not at all, specifically I think we should resolve
references if the user calls C-c ' with a prefix argument. Does that
sound reasonable? [Eric]
Yes [Dan]
** TODO fully purge org-babel-R of direct comint interaction
try to remove all code under the [[file:lisp/org-babel-R.el::functions%20for%20evaluation%20of%20R%20code][;; functions for evaluation of R code]] line
** TODO improve the source-block snippet
[[file:~/src/emacs-starter-kit/src/snippets/text-mode/rst-mode/chap::name%20Chapter%20title][file:~/src/emacs-starter-kit/src/snippets/text-mode/rst-mode/chap::name Chapter title]]
#+begin_example
,#name : Chapter title
,# --
${1:Chapter}
${1:$(make-string (string-width text) ?\=)}
$0
#+end_example
[[file:snippets/org-mode/sb][sb -- snippet]]
waiting for guidance from those more familiar with yasnippets
** TODO resolve references to other buffers
This would allow source blocks to call upon tables, source-blocks,
and results in other buffers.
See...
- [[file:lisp/org-babel-ref.el::TODO%20allow%20searching%20for%20names%20in%20other%20buffers][org-babel-ref.el:searching-in-other-buffers]]
- [[file:lisp/org-babel.el::defun%20org-babel%20find%20named%20result%20name][org-babel.el#org-babel-find-named-result]]
** TODO figure out how to handle graphic output
This is listed under [[* graphical output][graphical output]] in out objectives.
This should take advantage of the =:results file= option, and
languages which almost always produce graphical output should set
=:results file= to true by default. That would handle placing these
results in the buffer. Then if there is a combination of =silent= and
=file= =:results= headers we could drop the results to a temp buffer
and pop open that buffer...
** TODO share org-babel
how should we share org-babel?
- post to org-mode and ess mailing lists
- create a org-babel page on worg
- create a short screencast demonstrating org-babel in action
*** examples
we need to think up some good examples
**** interactive tutorials
This could be a place to use [[* org-babel assertions][org-babel assertions]].
for example the first step of a tutorial could assert that the version
of the software-package (or whatever) is equal to some value, then
source-code blocks could be used with confidence (and executed
directly from) the rest of the tutorial.
**** answering a text-book question w/code example
org-babel is an ideal environment enabling both the development and
demonstrationg of the code snippets required as answers to many
text-book questions.
**** something using tables
maybe something along the lines of calculations from collected grades
**** file sizes
Maybe something like the following which outputs sizes of directories
under the home directory, and then instead of the trivial =emacs-lisp=
block we could use an R block to create a nice pie chart of the
results.
#+srcname: sizes
#+begin_src bash :results replace
du -sc ~/*
#+end_src
#+begin_src emacs-lisp :var sizes=sizes :results replace
(mapcar #'car sizes)
#+end_src
** TODO command line execution
Allow source code blocks to be called form the command line. This
will be easy using the =sbe= function in [[file:lisp/org-babel-table.el][org-babel-table.el]].
This will rely upon [[* resolve references to other buffers][resolve references to other buffers]].
** TODO inline source code blocks [3/5]
Like the =\R{ code }= blocks
not sure what the format should be, maybe just something simple
like =src_lang[]{}= where lang is the name of the source code
language to be evaluated, =[]= is optional and contains any header
arguments and ={}= contains the code.
(see [[* (sandbox) inline source blocks][the-sandbox]])
*** DONE evaluation with \C-c\C-c
Putting aside the header argument issue for now we can just run these
with the following default header arguments
- =:results= :: silent
- =:exports= :: results
*** DONE inline exportation
Need to add an interblock hook (or some such) through org-exp-blocks
*** DONE header arguments
We should make it possible to use header arguments.
*** TODO fontification
we should color these blocks differently
*** TODO refine html exportation
should use a span class, and should show original source in tool-tip
** TODO allow tables with hline to be passed as args into R
This doesn't seem to work at the moment (example below). It would
also be nice to have a natural way for the column names of the org
table to become the column names of the R data frame, and to have
the option to specify that the first column is to be used as row
names in R (these must be unique). But this might require a bit of
thinking about.
#+TBLNAME: egtable
| col1 | col2 | col3 |
|------+---------+------|
| 1 | 2 | 3 |
| 4 | schulte | 6 |
#+TBLNAME: egtable2
| 1 | 2 | 3 |
| 4 | schulte | 6 |
#+begin_src R var tabel=egtable
tabel
#+end_src
#+resname:
| "col1" | "col2" | "col3" |
|--------+-----------+--------|
| 1 | 2 | 3 |
| 4 | "schulte" | 6 |
Another example is in the [[*operations%20in%20on%20tables][grades example]].
** PROPOSED conversion between org-babel and noweb (e.g. .Rnw) format
I haven't thought about this properly. Just noting it down. What
Sweave uses is called "R noweb" (.Rnw).
I found a good description of noweb in the following article (see
the [[http://www.cs.tufts.edu/~nr/pubs/lpsimp.pdf][pdf]]).
I think there are two parts to noweb, the construction of
documentation and the extraction of source-code (with notangle).
*documentation*: org-mode handles all of our documentation needs in
a manner that I believe is superior to noweb.
*source extraction* At this point I don't see anyone writing large
applications with 100% of the source code contained in org-babel
files, rather I see org-babel files containing things like
- notes with active code chunks
- interactive tutorials
- requirements documents with code running test suites
- and of course experimental reports with the code to run the
experiment, and perform analysis
Basically I think the scope of the programs written in org-babel
(at least initially) will be small enough that it wont require the
addition of a tangle type program to extract all of the source code
into a running application.
On the other hand, since we already have named blocks of source
code which reference other blocks on which they rely, this
shouldn't be too hard to implement either on our own, or possibly
relying on something like noweb/notangle.
** PROPOSED support for passing paths to files between source blocks
Maybe this should be it's own result type (in addition to scalars and
vectors). The reason being that some source-code blocks (for example
ditaa or anything that results in the creation of a file) may want to
pass a file path back to org-mode which could then be inserted into
the org-mode buffer as a link to the file...
This would allow for display of images upon export providing
functionality similar to =org-exp-blocks= only in a more general
manner.
** PROPOSED re-implement helper functions from org-R
Much of the power of org-R seems to be in it's helper functions for
the quick graphing of tables. Should we try to re-implement these
functions on top of org-babel?
I'm thinking this may be useful both to add features to org-babel-R and
also to potentially suggest extensions of the framework. For example
one that comes to mind is the ability to treat a source-code block
like a function which accepts arguments and returns results. Actually
this can be it's own TODO (see [[* source blocks as functions][source blocks as functions]]).
** DEFERRED use textConnection to pass tsv to R?
When passing args from the org buffer to R, the following route is
used: arg in buffer -> elisp -> tsv on file -> data frame in R. I
think it would be possible to avoid having to write to file by
constructing an R expression in org-babel-R-assign-elisp, something
like this
#+begin_src emacs-lisp
(org-babel-R-input-command
(format "%s <- read.table(textConnection(\"%s\"), sep=\"\\t\", as.is=TRUE)"
name (orgtbl-to-tsv value '(:sep "\t" :fmt org-babel-R-quote-tsv-field))))
#+end_src
I haven't tried to implement this yet as it's basically just
fiddling with something that works. The only reason for it I can
think of would be efficiency and I haven't tested that.
This Didn't work after an initial test. I still think this is a
good idea (I also think we should try to do something similar when
writing out results frmo R to elisp) however as it wouldn't result
in any functional changes I'm bumping it down to deferred for
now. [Eric]
for quick tests
#+tblname: quick-test
| 1 | 2 | 3 |
#+srcname: quick-test-src-blk
#+begin_src R :var vec=quick-test
mean(mean(vec))
#+end_src
: 2
** DEFERRED re-implement R evaluation using ess-command or ess-execute
I don't have any complaints with the current R evaluation code or
behaviour, but I think it would be good to use the ESS functions
from a political point of view. Plus of course it has the normal
benefits of an API (insulates us from any underlying changes etc). [DED]
I'll look into this. I believe that I looked at and rejected these
functions initially but now I can't remember why. I agree with
your overall point about using API's where available. I will take
a look back at these and either switch to using the ess commands,
or at least articulate under this TODO the reasons for using our
custom R-interaction commands. [Eric]
ess-execute
Lets just replace =org-babel-R-input-command= with =ess-execute=.
I tried this, and although it works in some situations, I find that
=ess-command= will often just hang indefinitely without returning
results. Also =ess-execute= will occasionally hang, and pops up
the buffer containing the results of the command's execution, which
is undesirable. For now these functions can not be used. Maybe
someone more familiar with the ESS code can recommend proper usage
of =ess-command= or some other lower-level function which could be
used in place of [[file:lisp/org-babel-R.el::defun%20org-babel%20R%20input%20command%20command][org-babel-R-input-command]].
*** ess functions
#+begin_quote ess-command
(ess-command COM &optional BUF SLEEP NO-PROMPT-CHECK)
Send the ESS process command COM and delete the output
from the ESS process buffer. If an optional second argument BUF exists
save the output in that buffer. BUF is erased before use.
COM should have a terminating newline.
Guarantees that the value of .Last.value will be preserved.
When optional third arg SLEEP is non-nil, `(sleep-for (* a SLEEP))'
will be used in a few places where `a' is proportional to `ess-cmd-delay'.
#+end_quote
#+begin_quote ess-execute
(ess-execute COMMAND &optional INVERT BUFF MESSAGE)
Send a command to the ESS process.
A newline is automatically added to COMMAND. Prefix arg (or second arg
INVERT) means invert the meaning of
`ess-execute-in-process-buffer'. If INVERT is 'buffer, output is
forced to go to the process buffer. If the output is going to a
buffer, name it *BUFF*. This buffer is erased before use. Optional
fourth arg MESSAGE is text to print at the top of the buffer (defaults
to the command if BUFF is not given.)
#+end_quote
*** out current setup
1) The body of the R source code block is wrapped in a function
2) The function is called inside of a =write.table= function call
writing the results to a table
3) The table is read using =org-table-import=
** DEFERRED Rework Interaction with Running Processes [0/3]
*** TODO ability to select which of multiple sessions is being used
Increasingly it is looking like we're going to want to run all
source code blocks in comint buffer (sessions). Which will have
the benefits of
1) allowing background execution
2) maintaining state between source-blocks
- allowing inline blocks w/o header arguments
**** R sessions
(like ess-switch-process in .R buffers)
Maybe this could be packaged into a header argument, something
like =:R_session= which could accept either the name of the
session to use, or the string =prompt=, in which case we could use
the =ess-switch-process= command to select a new process.
*** TODO evaluation of shell code as background process?
After C-c C-c on an R code block, the process may appear to
block, but C-g can be used to reclaim control of the .org buffer,
without interrupting the R evalution. However I believe this is not
true of bash/sh evaluation. [Haven't tried other languages] Perhaps
a solution is just to background the individual shell commands.
The other languages (aside from emacs lisp) are run through the
shell, so if we find a shell solution it should work for them as
well.
Adding an ampersand seems to be a supported way to run commands in
the background (see [[http://www.emacswiki.org/emacs/ExecuteExternalCommand#toc4][external-commands]]). Although a more extensible
solution may involve the use of the [[elisp:(progn (describe-function 'call-process-region) nil)][call-process-region]] function.
Going to try this out in a new file [[file:lisp/org-babel-proc.el][org-babel-proc.el]]. This should
contain functions for asynchronously running generic shell commands
in the background, and then returning their input.
**** partial update of org-mode buffer
The sleekest solution to this may be using a comint buffer, and
then defining a filter function which would incrementally interpret
the results as they are returned, including insertion into the
org-mode buffer. This may actually cause more problems than it is
worth, what with the complexities of identifying the types of
incrementally returned results, and the need for maintenance of a
process marker in the org buffer.
**** 'working' spinner
It may be nice and not too difficult to place a spinner on/near the
evaluating source code block
*** TODO conversion of output from interactive shell, R (and python) sessions to org-babel buffers
[DED] This would be a nice feature I think. Although an org-babel
purist would say that it's working the wrong way round... After
some interactive work in a *R* buffer, you save the buffer, maybe
edit out some lines, and then convert it to org-babel format for
posterity. Same for a shell session either in a *shell* buffer, or
pasted from another terminal emulator. And python of course.
** DONE Remove protective commas from # comments before evaluating
org inserts protective commas in front of ## comments in language
modes that use them. We need to remove them prior to sending code
to the interpreter.
#+srcname: testing-removal-of-protective-comas
#+begin_src ruby
,# this one might break it??
:comma_protection
#+end_src
** DONE pass multiple reference arguments into R
Can we do this? I wasn't sure how to supply multiple 'var' header
args. Just delete this if I'm being dense.
This should be working, see the following example...
#+srcname: two-arg-example
#+begin_src R :var n=2 :var m=8
n + m
#+end_src
#+resname: two-arg-example
: 10
** DONE ensure that table ranges work
when a table range is passed to org-babel as an argument, it should be
interpreted as a vector.
| 1 | 2 | simple |
| 2 | 3 | Fixnum:1 |
| 3 | 4 | Array:123456 |
| 4 | 5 | |
| 5 | 6 | |
| 6 | 7 | |
#+TBLFM: @1$3='(sbe simple-sbe-example (n 4))::@2$3='(sbe task-table-range (n @1$1..@6$1))::@3$3='(sbe task-table-range (n (@1$1..@6$1)))
#+srcname: simple-sbe-example
#+begin_src emacs-lisp
"simple"
#+end_src
#+srcname: task-table-range
#+begin_src ruby :var n=simple-sbe-example
"#{n.class}:#{n}"
#+end_src
#+srcname: simple-results
#+begin_src emacs-lisp :var n=task-table-range(n=(1 2 3))
n
#+end_src
#+resname: simple-results
: Array:123
#+srcname: task-arr-referent
#+begin_src ruby :var ar=(1 2 3)
ar.size
#+end_src
#+resname: task-arr-referent
: 3
** DONE global variable indicating default to vector output
how about an alist... =org-babel-default-header-args= this may already
exist... just execute the following and all source blocks will default
to vector output
#+begin_src emacs-lisp
(setq org-babel-default-header-args '((:results . "vector")))
#+end_src
** DONE name named results if source block is named
currently this isn't happening although it should be
#+srcname: test-naming-named-source-blocks
#+begin_src emacs-lisp
:namer
#+end_src
#+resname: test-naming-named-source-blocks
: :namer
** DONE (simple caching) check for named results before source blocks
see the TODO comment in [[file:lisp/org-babel-ref.el::TODO%20This%20should%20explicitly%20look%20for%20resname%20lines%20before][org-babel-ref.el#org-babel-ref-resolve-reference]]
** DONE set =:results silent= when eval with prefix argument
#+begin_src emacs-lisp
'silentp
#+end_src
** DONE results-type header (vector/file) [3/3]
In response to a point in Dan's email. We should allow the user to
force scalar or vector results. This could be done with a header
argument, and the default behavior could be controlled through a
configuration variable.
#+srcname: task-trivial-vector
#+begin_src ruby :results replace vector
:scalar
#+end_src
#+resname:
| ":scalar" |
since it doesn't make sense to turn a vector into a scalar, lets
just add a two values...
- vector :: forces the results to be a vector (potentially 1 dimensional)
- file :: this throws an error if the result isn't a string, and
tries to treat it as a path to a file.
I'm just going to cram all of these into the =:results= header
argument. Then if we allow multiple header arguments it should
work out, for example one possible header argument string could be
=:results replace vector file=, which would *replace* any existing
results forcing the results into an org-mode table, and
interpreting any strings as file paths.
*** DONE multiple =:results= headers
#+srcname: multiple-result-headers
#+begin_src ruby :results replace silent
:schulte
#+end_src
#+resname:
*** DONE file result types
When inserting into an org-mode buffer create a link with the path
being the value, and optionally the display being the
=file-name-nondirectory= if it exists.
#+srcname: task-file-result
#+begin_src python :results replace file
"something"
#+end_src
#+resname:
[[something][something]]
This will be useful because blocks like =ditaa= and =dot= can return
the string path of their files, and can add =file= to their results
header.
*** DONE vector result types
#+srcname: task-force-results
#+begin_src emacs-lisp :results vector
8
#+end_src
#+resname:
| 8 |
** DONE results name
In order to do this we will need to start naming our results.
Since the source blocks are named with =#+srcname:= lines we can
name results with =#+resname:= lines (if the source block has no
name then no name is given to the =#+resname:= line on creation,
otherwise the name of the source block is used).
This will have the additional benefit of allowing results and
source blocks to be located in different places in a buffer (and
eventually in different buffers entirely).
#+srcname: developing-resnames
#+begin_src emacs-lisp :results silent
'schulte
#+end_src
Once source blocks are able to find their own =#+resname:= lines
we then need to...
#+srcname: sbe-w-new-results
#+begin_src emacs-lisp :results replace
(sbe "developing-resnames")
#+end_src
#+resname:
: schulte
*** TODO change the results insertion functions to use these lines
*** TODO teach references to resolve =#+resname= lines.
** DONE org-babel tests org-babel [1/1]
since we are accumulating this nice collection of source-code blocks
in the sandbox section we should make use of them as unit tests.
What's more, we should be able to actually use org-babel to run these
tests.
We would just need to cycle over every source code block under the
sandbox, run it, and assert that the return value is equal to what we
expect.
I have the feeling that this should be possible using only org-babel
functions with minimal or no additional elisp. It would be very cool
for org-babel to be able to test itself.
This is now done, see [[* Tests]].
*** DEFERRED org-babel assertions (may not be necessary)
These could be used to make assertions about the results of a
source-code block. If the assertion fails then the point could be
moved to the block, and error messages and highlighting etc... could
ensue
** DONE make C-c C-c work anywhere within source code block?
This seems like it would be nice to me, but perhaps it would be
inefficient or ugly in implementation? I suppose you could search
forward, and if you find #+end_src before you find #+begin_src,
then you're inside one. [DED]
Agreed, I think inside of the =#+srcname: line= would be useful as
well.
#+srcname: testing-out-cc
#+begin_src emacs-lisp
'schulte
#+end_src
** DONE integration with org tables
We should make it easy to call org-babel source blocks from org-mode
table formulas. This is practical now that it is possible to pass
arguments to org-babel source blocks.
See the related [[* (sandbox) integration w/org tables][sandbox]] header for tests/examples.
*** digging in org-table.el
In the past [[file:~/src/org/lisp/org-table.el::org%20table%20el%20The%20table%20editor%20for%20Org%20mode][org-table.el]] has proven difficult to work with.
Should be a hook in [[file:~/src/org/lisp/org-table.el::defun%20org%20table%20eval%20formula%20optional%20arg%20equation][org-table-eval-formula]].
Looks like I need to change this [[file:~/src/org/lisp/org-table.el::if%20lispp][if statement]] (line 2239) into a cond
expression.
** DONE source blocks as functions
Allow source code blocks to be called like functions, with arguments
specified. We are already able to call a source-code block and assign
it's return result to a variable. This would just add the ability to
specify the values of the arguments to the source code block assuming
any exist. For an example see
When a variable appears in a header argument, how do we differentiate
between it's value being a reference or a literal value? I guess this
could work just like a programming language. If it's escaped or in
quotes, then we count it as a literal, otherwise we try to look it up
and evaluate it.
** DONE folding of code blocks? [2/2]
[DED] In similar way to using outline-minor-mode for folding function
bodies, can we fold code blocks? #+begin whatever statements are
pretty ugly, and in any case when you're thinking about the overall
game plan you don't necessarily want to see the code for each Step.
*** DONE folding of source code block
Sounds good, and wasn't too hard to implement. Code blocks should
now be fold-able in the same manner as headlines (by pressing TAB
on the first line).
*** REJECTED folding of results
So, lets do a three-stage tab cycle... First fold the src block,
then fold the results, then unfold.
There's no way to tell if the results are a table or not w/o
actually executing the block which would be too expensive of an
operation.
** DONE selective export of text, code, figures
[DED] The org-babel buffer contains everything (code, headings and
notes/prose describing what you're up to, textual/numeric/graphical
code output, etc). However on export to html / LaTeX one might want
to include only a subset of that content. For example you might
want to create a presentation of what you've done which omits the
code.
[EMS] So I think this should be implemented as a property which can
be set globally or on the outline header level (I need to review
the mechanics of org-mode properties). And then as a source block
header argument which will apply only to a specific source code
block. A header argument of =:export= with values of
- =code= :: just show the code in the source code block
- =none= :: don't show the code or the results of the evaluation
- =results= :: just show the results of the code evaluation (don't
show the actual code)
- =both= :: show both the source code, and the results
this will be done in [[* (sandbox) selective export][(sandbox) selective export]].
** DONE a header argument specifying silent evaluation (no output)
This would be useful across all types of source block. Currently
there is a =:replace t= option to control output, this could be
generalized to an =:output= option which could take the following
options (maybe more)
- =t= :: this would be the default, and would simply insert the
results after the source block
- =replace= :: to replace any results which may already be there
- =silent= :: this would inhibit any insertion of the results
This is now implemented see the example in the [[* silent evaluation][sandbox]]
** DONE assign variables from tables in R
This is now working (see [[* (sandbox table) R][(sandbox-table)-R]]). Although it's not that
impressive until we are able to print table results from R.
** DONE insert 2-D R results as tables
everything is working but R and shell
*** DONE shells
*** DONE R
This has already been tackled by Dan in [[file:existing_tools/org-R.el::defconst%20org%20R%20write%20org%20table%20def][org-R:check-dimensions]]. The
functions there should be useful in combination with [[http://cran.r-project.org/doc/manuals/R-data.html#Export-to-text-files][R-export-to-csv]]
as a means of converting multidimensional R objects to emacs lisp.
It may be as simple as first checking if the data is multidimensional,
and then, if so using =write= to write the data out to a temporary
file from which emacs can read the data in using =org-table-import=.
Looking into this further, is seems that there is no such thing as a
scalar in R [[http://tolstoy.newcastle.edu.au/R/help/03a/3733.html][R-scalar-vs-vector]] In that light I am not sure how to
deal with trivial vectors (scalars) in R. I'm tempted to just treat
them as vectors, but then that would lead to a proliferation of
trivial 1-cell tables...
** DONE allow variable initialization from source blocks
Currently it is possible to initialize a variable from an org-mode
table with a block argument like =table=sandbox= (note that the
variable doesn't have to named =table=) as in the following example
#+TBLNAME: sandbox
| 1 | 2 | 3 |
| 4 | schulte | 6 |
#+begin_src emacs-lisp :var table=sandbox :results replace
(message (format "table = %S" table))
#+end_src
: "table = ((1 2 3) (4 \"schulte\" 6))"
It would be good to allow initialization of variables from the results
of other source blocks in the same manner. This would probably
require the addition of =#+SRCNAME: example= lines for the naming of
source blocks, also the =table=sandbox= syntax may have to be expanded
to specify whether the target is a source code block or a table
(alternately we could just match the first one with the given name
whether it's a table or a source code block).
At least initially I'll try to implement this so that there is no need
to specify whether the reference is to a table or a source-code block.
That seems to be simpler both in terms of use and implementation.
This is now working for emacs-lisp, ruby and python (and mixtures of
the three) source blocks. See the examples in the [[* (sandbox) referencing other source blocks][sandbox]].
This is currently working only with emacs lisp as in the following
example in the [[* emacs lisp source reference][emacs lisp source reference]].
** TODO Add languages [0/5]
I'm sure there are many more that aren't listed here. Please add
them, and bubble any that you particularly care about up to the top.
Any new language should be implemented in a org-babel-lang.el file.
Follow the pattern set by [[file:lisp/org-babel-script.el][org-babel-script.el]], [[file:lisp/org-babel-shell.el][org-babel-shell.el]] and
[[file:lisp/org-babel-R.el][org-babel-R.el]].
*** TODO perl
This could probably be added to [[file:lisp/org-babel-script.el][org-babel-script.el]]
*** TODO java
*** TODO ditaa
(see [[* file result types][file result types]])
*** TODO dot
(see [[* file result types][file result types]])
*** TODO asymptote
(see [[* file result types][file result types]])
* Bugs [11/14]
** TODO non-orgtbl formatted lists
for example
#+srcname: this-doesn't-match-orgtbl
#+begin_src emacs-lisp :results replace
'((:results . "replace"))
#+end_src
#+resname: this-doesn't-match-orgtbl
** TODO collapsing consecutive newlines in string output
#+srcname: multi-line-string-output
#+begin_src ruby :results replace
"the first line ends here
and this is the second one
even a third"
#+end_src
#+resname:
: the first line ends here
: and this is the second one
: return even a third
** TODO cursor movement when evaluating source blocks
E.g. the pie chart example. Despite the save-window-excursion in
org-babel-execute:R. (I never learned how to do this properly: org-R
jumps all over the place...)
** DONE R-code broke on "org-babel" rename
#+srcname: bug-R-babels
#+begin_src R
8 * 2
#+end_src
** DONE error on trivial R results
So I know it's generally not a good idea to squash error without
handling them, but in this case the error almost always means that
there was no file contents to be read by =org-table-import=, so I
think it's ok.
#+srcname: bug-trivial-r1
#+begin_src R :results replace
pie(c(1, 2, 3), labels = c(1, 2, 3))
#+end_src
#+srcname: bug-trivial-r2
#+begin_src R :results replace
8
#+end_src
#+resname: bug-trivial-r2
: 8
#+srcname: bug-trivial-r3
#+begin_src R :results replace
c(1,2,3)
#+end_src
#+resname: bug-trivial-r3
| 1 |
| 2 |
| 3 |
** DONE ruby new variable creation (multi-line ruby blocks)
Actually it looks like we were dropping all but the last line.
#+srcname: multi-line-ruby-test
#+begin_src ruby :var table=bug-numerical-table :results replace
total = 0
table.each{|n| total += n}
total/table.size
#+end_src
#+resname:
: 2
** DONE R code execution seems to choke on certain inputs
Currently the R code seems to work on vertical (but not landscape)
tables
#+srcname: little-fake
#+begin_src emacs-lisp
"schulte"
#+end_src
#+begin_src R :var num=little-fake
num
#+end_src
#+resname:
: schulte
: 11
: 11
: 11
: schulte
: 9
: 9
: 11
#+srcname: set-debug-on-error
#+begin_src emacs-lisp :results silent
(setq debug-on-error t)
#+end_src
#+srcname: bug-numerical-table
#+begin_src emacs-lisp :results silent
'(1 2 3)
#+end_src
#+srcname: bug-R-number-evaluation
#+begin_src R :var table=bug-numerical-table :results replace
mean(mean(table))
#+end_src
#+resname:
: 2
#+tblname: bug-vert-table
| 1 |
| 2 |
| 3 |
#+srcname: bug-R-vertical-table
#+begin_src R :var table=bug-vert-table :results silent
mean(table)
#+end_src
** DEFERRED org bug/request: prevent certain org behaviour within code blocks
E.g. [[]] gets recognised as a link (when there's text inside the
brackets). This is bad for R code at least, and more generally
could be argued to be inappropriate. Is it difficult to get org to
ignore text in code blocks? [DED]
I believe Carsten addressed this recently on the mailing list with
the comment that it was indeed a difficult issue. I believe this
may be one area where we could wait for an upstream (org-mode) fix.
** DONE with :results replace, non-table output doesn't replace table output
And vice versa. E.g. Try this first with table and then with len(table) [DED]
#+begin_src python :var table=sandbox :results replace
table
#+end_src
| 1 | 2 | 3 |
| 4 | "schulte" | 6 |
: 2
Yes, this is certainly a problem. I fear that if we begin replacing
anything immediately following a source block (regardless of whether
it matches the type of our current results) we may accidentally delete
hand written portions of the user's org-mode buffer.
I think that the best solution here would be to actually start
labeling results with a line that looks something like...
#+results: name
This would have a couple of benefits...
1) we wouldn't have to worry about possibly deleting non-results
(which is currently an issue)
2) we could reliably replace results even if there are different types
3) we could reference the results of a source-code block in variable
definitions, which would be useful if for example we don't wish to
re-run a source-block every time because it is long-running.
Thoughts? If no-one objects, I believe I will implement the labeling
of results.
** DONE extra quotes for nested string
Well R appears to be reading the tables without issue...
these *should* be quoted
#+srcname: ls
#+begin_src sh :results replace
ls
#+end_src
| "COPYING" |
| "README.markdown" |
| "block" |
| "examples.org" |
| "existing_tools" |
| "intro.org" |
| "org-babel" |
| "rorg.org" |
| "test-export.html" |
| "test-export.org" |
#+srcname: test-quotes
#+begin_src ruby :var tab=ls
tab[1][0]
#+end_src
: README.markdown
#+srcname: test-quotes
#+begin_src R :var tab=ls
as.matrix(tab[2,])
#+end_src
: README.markdown
** DONE simple ruby arrays not working
As an example eval the following. Adding a line to test
#+srcname: simple-ruby-array
#+begin_src ruby
[3, 4, 5]
#+end_src
#+srcname: ruby-array-test
#+begin_src ruby :var ar = simple-ruby-array
ar.first
#+end_src
** DONE space trailing language name
fix regexp so it works when there's a space trailing the language name
#+srcname: test-trailing-space
#+begin_src ruby
:schulte
#+end_src
** DONE Args out of range error
The following block resulted in the error below [DED]. It ran without
error directly in the shell.
#+begin_src sh
cd ~/work/genopca
for platf in ill aff ; do
for pop in CEU YRI ASI ; do
rm -f $platf/hapmap-genos-$pop-all $platf/hapmap-rs-all
cat $platf/hapmap-genos-$pop-* > $platf/hapmap-genos-$pop-all
cat $platf/hapmap-rs-* > $platf/hapmap-rs-all
done
done
#+end_src
executing source block with sh...
finished executing source block
string-equal: Args out of range: "", -1, 0
the error =string-equal: Args out of range: "", -1, 0= looks like what
used to be output when the block returned an empty results string.
This should be fixed in the current version, you should now see the
following message =no result returned by source block=.
** DONE ruby arrays not recognized as such
Something is wrong in [[file:lisp/org-babel-script.el]] related to the
recognition of ruby arrays as such.
#+begin_src ruby :results replace
[1, 2, 3, 4]
#+end_src
| 1 | 2 | 3 | 4 |
#+begin_src python :results replace
[1, 2, 3, 4]
#+end_src
| 1 | 2 | 3 | 4 |
* Tests
Evaluate all the cells in this table for a comprehensive test of the
org-babel functionality.
*Note*: if you have customized =org-babel-default-header-args= then some
of these tests may fail.
#+TBLNAME: org-babel-tests
| functionality | block | arg | expected | results | pass |
|-------------------------+----------------------------+-----+-------------+-------------+------|
| basic evaluation | | | | | pass |
|-------------------------+----------------------------+-----+-------------+-------------+------|
| emacs lisp | basic-elisp | | 5 | 5 | pass |
| shell | basic-shell | | 6 | 6 | pass |
| ruby | basic-ruby | | org-babel | org-babel | pass |
| python | basic-python | | hello world | hello world | pass |
| R | basic-R | | 13 | 13 | pass |
|-------------------------+----------------------------+-----+-------------+-------------+------|
| tables | | | | | pass |
|-------------------------+----------------------------+-----+-------------+-------------+------|
| emacs lisp | table-elisp | | 3 | 3 | pass |
| ruby | table-ruby | | 1-2-3 | 1-2-3 | pass |
| python | table-python | | 5 | 5 | pass |
| R | table-R | | 3.5 | 3.5 | pass |
|-------------------------+----------------------------+-----+-------------+-------------+------|
| source block references | | | | | pass |
|-------------------------+----------------------------+-----+-------------+-------------+------|
| all languages | chained-ref-last | | Array | Array | pass |
|-------------------------+----------------------------+-----+-------------+-------------+------|
| source block functions | | | | | pass |
|-------------------------+----------------------------+-----+-------------+-------------+------|
| emacs lisp | defun-fibb | | fibbd | fibbd | pass |
| run over | Fibonacci | 0 | 1 | 1 | pass |
| a | Fibonacci | 1 | 1 | 1 | pass |
| variety | Fibonacci | 2 | 2 | 2 | pass |
| of | Fibonacci | 3 | 3 | 3 | pass |
| different | Fibonacci | 4 | 5 | 5 | pass |
| arguments | Fibonacci | 5 | 8 | 8 | pass |
|-------------------------+----------------------------+-----+-------------+-------------+------|
| bugs and tasks | | | | | pass |
|-------------------------+----------------------------+-----+-------------+-------------+------|
| simple ruby arrays | ruby-array-test | | 3 | 3 | pass |
| R number evaluation | bug-R-number-evaluation | | 2 | 2 | pass |
| multi-line ruby blocks | multi-line-ruby-test | | 2 | 2 | pass |
| forcing vector results | test-forced-vector-results | | Array | Array | pass |
#+TBLFM: $5='(if (= (length $3) 1) (progn (message (format "running %S" '(sbe $2 (n $3)))) (sbe $2 (n $3))) (sbe $2))::$6='(if (string= $4 $5) "pass" (format "expected %S but was %S" $4 $5))
** basic tests
#+srcname: basic-elisp
#+begin_src emacs-lisp :results silent
(+ 1 4)
#+end_src
#+srcname: basic-shell
#+begin_src sh :results silent
expr 1 + 5
#+end_src
#+srcname: basic-ruby
#+begin_src ruby :results silent
"org-babel"
#+end_src
#+srcname: basic-python
#+begin_src python :results silent
'hello world'
#+end_src
#+srcname: basic-R
#+begin_src R :results silent
b <- 9
b + 4
#+end_src
** read tables
#+tblname: test-table
| 1 | 2 | 3 |
| 4 | 5 | 6 |
#+srcname: table-elisp
#+begin_src emacs-lisp :results silent :var table=test-table
(length (car table))
#+end_src
#+srcname: table-ruby
#+begin_src ruby :results silent :var table=test-table
table.first.join("-")
#+end_src
#+srcname: table-python
#+begin_src python :var table=test-table
table[1][1]
#+end_src
#+srcname: table-R
#+begin_src R :var table=test-table
mean(mean(table))
#+end_src
** references
Lets pass a references through all of our languages...
Lets start by reversing the table from the previous examples
#+srcname: chained-ref-first
#+begin_src python :var table = test-table
table.reverse()
table
#+end_src
#+resname: chained-ref-first
| 4 | 5 | 6 |
| 1 | 2 | 3 |
Take the first part of the list
#+srcname: chained-ref-second
#+begin_src R :var table = chained-ref-first
table[1]
#+end_src
#+resname: chained-ref-second
| 4 |
| 1 |
Turn the numbers into string
#+srcname: chained-ref-third
#+begin_src emacs-lisp :var table = chained-ref-second
(mapcar (lambda (el) (format "%S" el)) table)
#+end_src
#+resname: chained-ref-third
| "(4)" | "(1)" |
and Check that it is still a list
#+srcname: chained-ref-last
#+begin_src ruby :var table=chained-ref-third
table.class.name
#+end_src
** source blocks as functions
#+srcname: defun-fibb
#+begin_src emacs-lisp :results silent
(defun fibbd (n) (if (< n 2) 1 (+ (fibbd (- n 1)) (fibbd (- n 2)))))
#+end_src
#+srcname: fibonacci
#+begin_src emacs-lisp :results silent :var n=7
(fibbd n)
#+end_src
** sbe tests
Testing the insertion of results into org-mode tables.
#+srcname: multi-line-output
#+begin_src ruby :results replace
"the first line ends here
and this is the second one
even a third"
#+end_src
#+resname:
: the first line ends here
: and this is the second one
: return even a third
#+srcname: multi-line-error
#+begin_src ruby :results replace
raise "oh nooooooooooo"
#+end_src
#+resname:
: -:5: warning: parenthesize argument(s) for future version
: -:5:in `main': oh nooooooooooo (RuntimeError)
: from -:8
| the first line ends here... | -:5: warning: parenthesize argument(s) for future version... |
#+TBLFM: $1='(sbe "multi-line-output")::$2='(sbe "multi-line-error")
** forcing results types tests
#+srcname: test-trivial-vector
#+begin_src emacs-lisp :results vector silent
8
#+end_src
#+srcname: test-forced-vector-results
#+begin_src ruby :var triv=test-trivial-vector :results silent
triv.class.name
#+end_src
* Sandbox
:PROPERTIES:
:CUSTOM_ID: sandbox
:END:
To run these examples evaluate [[file:lisp/org-babel-init.el][org-babel-init.el]]
** org-babel.el beginning functionality
#+begin_src sh :results replace
date
#+end_src
: Thu May 14 18:52:25 EDT 2009
#+begin_src ruby
Time.now
#+end_src
: Thu May 14 18:59:09 -0400 2009
#+begin_src python
"Hello World"
#+end_src
: Hello World
** org-babel-R
#+begin_src R :results replace
a <- 9
b <- 16
a + b
#+end_src
: 25
#+begin_src R
hist(rgamma(20,3,3))
#+end_src
** org-babel plays with tables
Alright, this should demonstrate both the ability of org-babel to read
tables into a lisp source code block, and to then convert the results
of the source code block into an org table. It's using the classic
"lisp is elegant" demonstration transpose function. To try this
out...
1. evaluate [[file:lisp/org-babel-init.el]] to load org-babel and friends
2. evaluate the transpose definition =\C-c\C-c= on the beginning of
the source block
3. evaluate the next source code block, this should read in the table
because of the =:var table=previous=, then transpose the table, and
finally it should insert the transposed table into the buffer
immediately following the block
*** Emacs lisp
#+begin_src emacs-lisp :results silent
(defun transpose (table)
(apply #'mapcar* #'list table))
#+end_src
#+TBLNAME: sandbox
| 1 | 2 | 3 |
| 4 | schulte | 6 |
#+begin_src emacs-lisp :var table=sandbox :results replace
(transpose table)
#+end_src
#+begin_src emacs-lisp
'(1 2 3 4 5)
#+end_src
| 1 | 2 | 3 | 4 | 5 |
*** Ruby and Python
#+begin_src ruby :var table=sandbox :results replace
table.first.join(" - ")
#+end_src
: "1 - 2 - 3"
#+begin_src python :var table=sandbox :results replace
table[0]
#+end_src
| 1 | 2 | 3 |
#+begin_src ruby :var table=sandbox :results replace
table
#+end_src
| 1 | 2 | 3 |
| 4 | "schulte" | 6 |
#+begin_src python :var table=sandbox :results replace
len(table)
#+end_src
: 2
| "__add__" | "__class__" | "__contains__" | "__delattr__" | "__delitem__" | "__delslice__" | "__doc__" | "__eq__" | "__format__" | "__ge__" | "__getattribute__" | "__getitem__" | "__getslice__" | "__gt__" | "__hash__" | "__iadd__" | "__imul__" | "__init__" | "__iter__" | "__le__" | "__len__" | "__lt__" | "__mul__" | "__ne__" | "__new__" | "__reduce__" | "__reduce_ex__" | "__repr__" | "__reversed__" | "__rmul__" | "__setattr__" | "__setitem__" | "__setslice__" | "__sizeof__" | "__str__" | "__subclasshook__" | "append" | "count" | "extend" | "index" | "insert" | "pop" | "remove" | "reverse" | "sort" |
*** (sandbox table) R
#+TBLNAME: sandbox_r
| 1 | 2 | 3 |
| 4 | schulte | 6 |
#+begin_src R :results replace
x <- c(rnorm(10, mean=-3, sd=1), rnorm(10, mean=3, sd=1))
x
#+end_src
| -3.35473133869346 |
| -2.45714878661 |
| -3.32819924928633 |
| -2.97310212756194 |
| -2.09640758369576 |
| -5.06054014378736 |
| -2.20713700711221 |
| -1.37618039712037 |
| -1.95839385821742 |
| -3.90407396475502 |
| 2.51168071590226 |
| 3.96753011570494 |
| 3.31793212627865 |
| 1.99829753972341 |
| 4.00403686419829 |
| 4.63723764452927 |
| 3.94636744261313 |
| 3.58355906547775 |
| 3.01563442274226 |
| 1.7634976849927 |
#+begin_src R var tabel=sandbox_r :results replace
tabel
#+end_src
| 1 | 2 | 3 |
| 4 | "schulte" | 6 |
*** shell
Now shell commands are converted to tables using =org-table-import=
and if these tables are non-trivial (i.e. have multiple elements) then
they are imported as org-mode tables...
#+begin_src sh :results replace
ls -l
#+end_src
| "total" | 208 | "" | "" | "" | "" | "" | "" |
| "-rw-r--r--" | 1 | "dan" | "dan" | 57 | 2009 | 15 | "block" |
| "-rw-r--r--" | 1 | "dan" | "dan" | 35147 | 2009 | 15 | "COPYING" |
| "-rw-r--r--" | 1 | "dan" | "dan" | 722 | 2009 | 18 | "examples.org" |
| "drwxr-xr-x" | 4 | "dan" | "dan" | 4096 | 2009 | 19 | "existing_tools" |
| "-rw-r--r--" | 1 | "dan" | "dan" | 2207 | 2009 | 14 | "intro.org" |
| "drwxr-xr-x" | 2 | "dan" | "dan" | 4096 | 2009 | 18 | "org-babel" |
| "-rw-r--r--" | 1 | "dan" | "dan" | 277 | 2009 | 20 | "README.markdown" |
| "-rw-r--r--" | 1 | "dan" | "dan" | 11837 | 2009 | 18 | "rorg.html" |
| "-rw-r--r--" | 1 | "dan" | "dan" | 61829 | 2009 | 19 | "#rorg.org#" |
| "-rw-r--r--" | 1 | "dan" | "dan" | 60190 | 2009 | 19 | "rorg.org" |
| "-rw-r--r--" | 1 | "dan" | "dan" | 972 | 2009 | 11 | "test-export.org" |
** silent evaluation
#+begin_src ruby
:im_the_results
#+end_src
: :im_the_results
#+begin_src ruby :results silent
:im_the_results
#+end_src
#+begin_src ruby :results replace
:im_the_results_
#+end_src
: :im_the_results_
** (sandbox) referencing other source blocks
Doing this in emacs-lisp first because it's trivial to convert
emacs-lisp results to and from emacs-lisp.
*** emacs lisp source reference
This first example performs a calculation in the first source block
named =top=, the results of this calculation are then saved into the
variable =first= by the header argument =:var first=top=, and it is
used in the calculations of the second source block.
#+SRCNAME: top
#+begin_src emacs-lisp
(+ 4 2)
#+end_src
#+begin_src emacs-lisp :var first=top :results replace
(* first 3)
#+end_src
: 18
This example is the same as the previous only the variable being
passed through is a table rather than a number.
#+begin_src emacs-lisp :results silent
(defun transpose (table)
(apply #'mapcar* #'list table))
#+end_src
#+TBLNAME: top_table
| 1 | 2 | 3 |
| 4 | schulte | 6 |
#+SRCNAME: second_src_example
#+begin_src emacs-lisp :var table=top_table
(transpose table)
#+end_src
#+begin_src emacs-lisp :var table=second_src_example :results replace
(transpose table)
#+end_src
| 1 | 2 | 3 |
| 4 | "schulte" | 6 |
*** ruby python
Now working for ruby
#+srcname: start
#+begin_src ruby
89
#+end_src
#+begin_src ruby :var other=start :results replace
2 * other
#+end_src
and for python
#+SRCNAME: start_two
#+begin_src python
98
#+end_src
#+begin_src python :var another=start_two :results replace
another*3
#+end_src
*** mixed languages
Since all variables are converted into Emacs Lisp it is no problem to
reference variables specified in another language.
#+SRCNAME: ruby-block
#+begin_src ruby
2
#+end_src
#+SRCNAME: lisp_block
#+begin_src emacs-lisp :var ruby-variable=ruby-block
(* ruby-variable 8)
#+end_src
#+begin_src python :var lisp_var=lisp_block
lisp_var + 4
#+end_src
: 20
*** R
#+srcname: first_r
#+begin_src R :results replace
a <- 9
a
#+end_src
: 9
#+begin_src R :var other=first_r :results replace
other + 2
#+end_src
: 11
** (sandbox) selective export
For exportation tests and examples see (including exportation of
inline source code blocks) [[file:test-export.org]]
** (sandbox) source blocks as functions
#+srcname: default
#+begin_src emacs-lisp :results silent
5
#+end_src
#+srcname: triple
#+begin_src emacs-lisp :var n=default :results replace
(* 3 n)
#+end_src
: 15
#+begin_src emacs-lisp :var result=triple(n=3, m=98) :results replace
result
#+end_src
: 294
The following just demonstrates the ability to assign variables to
literal values, which was not implemented until recently.
#+begin_src ruby :var num="eric" :results replace
num+" schulte "
#+end_src
: "eric schulte "
** (sandbox) inline source blocks
This is an inline source code block src_ruby{1 + 6}. And another
source block with text output src_emacs-lisp{"eric"}.
This is an inline source code block with header
arguments. src_ruby[:var n=fibbd( n = 0 )]{n}
** (sandbox) integration w/org tables
#+begin_src emacs-lisp :results silent
(defun fibbd (n) (if (< n 2) 1 (+ (fibbd (- n 1)) (fibbd (- n 2)))))
#+end_src
#+srcname: fibbd
#+begin_src emacs-lisp :var n=4 :results silent
(fibbd n)
#+end_src
#+begin_src emacs-lisp :results silent
(mapcar #'fibbd '(0 1 2 3 4 5 6 7 8))
#+end_src
Something is not working here. The function `sbe ' works fine when
called from outside of the table (see the source block below), but
produces an error when called from inside the table. I think there
must be some narrowing going on during intra-table emacs-lisp
evaluation.
| original | fibbd |
|----------+-------|
| 0 | 1 |
| 1 | 1 |
| 2 | 2 |
| 3 | 3 |
| 4 | 5 |
| 5 | 8 |
| 6 | 13 |
| 7 | 21 |
| 8 | 34 |
| 9 | 55 |
#+TBLFM: $2='(sbe "fibbd" (n $1))
silent-result
#+begin_src emacs-lisp :results silent
(sbe 'fibbd (n "8"))
#+end_src
* Buffer Dictionary
LocalWords: DBlocks dblocks org-babel el eric fontification
|