![[Home page]](../PythonPowered.gif "Home page"){kind=small}
| Books | Code | Blog | Python | Author | Train | Find | ©M.Lutz |
File: mergeall-android-scripts/_README.html
<HTML>
<!--DOCTYPE intentionally omitted for now: toolbar (and more?) rendering differs-->
<HEAD>
<!-- embedded Unicode symbols (in top 1k bytes) -->
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<!--only if online, the snake -->
<LINK rel="shortcut icon" type="image/x-icon" href="https://learning-python.com/favicon.ico" />
<!-- fit to window on mobiles -->
<META name="viewport" content="width=device-width, initial-scale=1.0">
<!-- plus analytics code, etc. -->
<!--
Anonymous analytics to prioritize work, enabled in online resources
only. Automatically inserted at publish time by insert-analytics.py.
-->
<!-- 1) Universal Analytics tag (custom): stops collecting data on Jul-1-2023 -->
<SCRIPT>
// Start async JS-file fetch, if not already cached
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
// Queue actions to run in order after async JS-file fetch finished
ga('create', 'UA-52579036-1', 'auto'); // Create tracker object (and queue)
ga('set', 'anonymizeIp', true); // Anonymize IP addr (&aip) [Jun-2019]
ga('send', 'pageview'); // Send page-view event now
</SCRIPT>
<!-- 2) Google Analytics 4 tag: added to site Oct-2022 (okay to keep UA tag) -->
<!-- Google tag (gtag.js) -->
<script async src="https://www.googletagmanager.com/gtag/js?id=G-J8CTEZHX3L"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-J8CTEZHX3L');
</script>
<!-- End analytics insert -->
<!-- =============================================================================== -->
<STYLE>
/*-------------------------------------------------------------------------------------
elements
--------------------------------------------------------------------------------------*/
BODY {
font-family: Arial, Helvetica, sans-serif; /* font for entire doc: precedence list */
}
BODY {
margin-left: 13px; /* entire doc; padding=inside a border, margin=outside */
margin-right: 13px; /* narrower lines read better, samsung edge warps text */
}
LI {
margin-bottom: 8px; /* for every <LI> here: easier to read and tap */
}
DT {
font-weight: bold; /* hanging-list title lines */
/*margin-bottom: 10px;*/
}
DD {
margin-top: 8px;
margin-bottom: 15px; /* hanging-list detail blocks */
}
H1, H2 {
background-color: #006699;
color: white;
padding-left: 2px; /* add space to left of title text */
}
H1 {
margin-top: 0px;
}
H2 {
margin-top: 50px;
margin-bottom: 30px;
}
H3 {
background-color: #d0ebff;
margin-top: 40px;
padding-left: 2px;
}
H3.cheat { /* because appendix B shouldn't blow up the toc */
background-color: white; /* later bailed on this when the appendix grew longer */
margin-top: 35px;
}
/*-------------------------------------------------------------------------------------
use main website's code-block style
--------------------------------------------------------------------------------------*/
PRE {
overflow-x: auto; /* scroll if needed for mobile and shrunken windows */
white-space: pre; /* just to be sure? */
font-family: Courier, monospace, sans-serif; /* better: else some mobiles aren't mono */
background-color: ivory; /* color, okay iff scrolled to window size (else truncated) */
outline: black solid 1px; /* [IE<=9 overrides ahead] border, okay if scrolled (else bisects code in most wb) */
padding: 5px; /* space around code within pre block, one=all 4 sides*/
margin: 20px;
}
PRE {
font-family: Consolas, "Ubuntu mono", monospace, Courier;
font-size: 95%;
}
@media screen and (max-width: 640px) {
PRE, pre.fancy {
font-size: 89%; /* scale down more on small views (~640 pixels): show more text */
}
}
/*-------------------------------------------------------------------------------------
use main website's inline-literal fonts
--------------------------------------------------------------------------------------*/
CODE {
font-family: Consolas, "Ubuntu mono", monospace, Courier;
font-size: 95%;
background-color: #eeeeee; /* very-light grey to accentuate; same as #eee */
}
@media screen and (max-width: 640px) {
CODE {
font-size: 89%; /* scale down more on small views (~640 pixels): show more text */
}
}
/* augment site <code> to avoid viewport overflows */
CODE {
word-break: break-all; /* else long paths overflow and break viewport */
} /* see note in android-deltas-sync/_README.html */
/*-------------------------------------------------------------------------------------
classes
--------------------------------------------------------------------------------------*/
/* Table of contents */
.tocmain {
margin-top: 5px;
margin-bottom: 12px;
font-weight: bold;
font-style: normal;
/*list-style-type: square;*/ /* not squares: bid and bad valign in IE <= 9 */
}
.tocnest {
margin-top: 5px;
font-weight: normal;
font-style: normal;
}
/* Sidebars (not updates) */
.notebox {
margin-left: 20px; /* match <pre> indents */
margin-right: 20px;
margin-top: 40px; /* but more space above to offset (was 30, 48) */
margin-bottom: 55px;
outline: black solid 1px; /* [IE<=9 overrides ahead] dropped border: see pre */
padding: 5px;
background-color: #f1fdff; /* apr18: colorize */
}
.noteboxseries {
margin-top: 50px; /* more space before first is > one notebox */
}
P.noteboxheader { /* 'header' line in sidebars */
font-weight: bold; /* now split off to own <p>, as in u-t-a */
font-style: italic; /* and .notebox used in a <div>, not <p> */
margin-top: 0px;
}
P.noteboxlast {
margin-bottom: 0px; /* final content: trim blank line at end (?) */
}
/* Updates (not sidebars) */
SPAN.update {
font-weight: bold;
font-style: italic;
color: black;
background-color: wheat; /* was ivory; make more obvious */
/*
border: thin solid
padding-left: 2px;
padding-right: 2px;
*/
}
/* unused */
SPAN.overstrike { /* meant for "Q"=>"R" but barely noticeable */
text-decoration: line-through; /* or use <del> or <s> in html 5 */
}
/* Wrap-ups (offset) */
P.summary {
margin-top: 35px;
}
/* Section-prefaces (offset) */
P.preface { /* sep20: not really a notebox or update */
/*margin: 20px;*/ /* used and needed to qualify docs that follow */
margin-left: 20px; /* match <pre> indents */
margin-right: 20px;
margin-top: 30px;
margin-bottom: 30px;
border: thin solid black; /* awfully busy, but such is life in thrashland */
padding: 5px;
}
SPAN.preface { /* style the "Preface" tag - or not */
font-weight: bold; /* originally used span.update, but allow diffs */
font-style: italic;
color: black;
background-color: #d0ebff; /* differ from updates */
}
/*-------------------------------------------------------------------------------------
per main website, do not scale up text fonts in landscape mode on iOS Safari
(else no more text is visible), but still allow user-initiated zooms
--------------------------------------------------------------------------------------*/
@media screen and (max-device-width: 640px) {
html { /* from the wildly adhoc department! */
-webkit-text-size-adjust: 100%; /* not 'none': prevents zoom in/out */
}
}
/*------------------------------------------------------------------------------------
[Nov23] Disable now-broken font size boosting on Android Chrome.
On websites with text, Android Chrome (AChrome) now regularly opens
pages with some text initially scaled up to larger system or browser
font-size settings... and then abruptly scales the very same text
down to a smaller default size after initial user scrolls. A reload
after scrolls restores the botched larger sizes--until more scrolls.
The CSS below makes this moot, by disabling AChrome text scaling
enough to avoid the new+bogus initial resizes. See site's _main.css.
------------------------------------------------------------------------------------*/
HTML { /* html and body work same (h contains b) */
text-size-adjust: none; /* chrome (all), edge, samsung, opera, etc */
-webkit-text-size-adjust: none; /* safari (ios, ipad default none, macos) */
-moz-text-size-adjust: none; /* firefox (not desktop) */
}
/*-------------------------------------------------------------------------------------
navbar, from Mergeall's user guide (which got it from main website)
--------------------------------------------------------------------------------------*/
.blocklinkbar {
display: block; /* toolbar buttons style */
width: 100%;
text-decoration: none;
color: white;
}
.blocklinkbar:hover { /* on toolbar hover, underline link */
text-decoration: underline;
}
.footerdiv {
overflow-x: auto; /*scroll*/ /* scroll + always show space (for URL hover popups) */
width: 100%; /* alt: overflow-x:auto adds scroll space iff needed */
left: 0px; /* no space on left for <body> indent */
position: fixed; /* anchor to bottom of display */
bottom: 0; /* position:fixed + scroll doesn't work for <table> */
}
.footertable { /* common browser-neutral formatting for footer button bar */
background-color: #006699; /* was .panel, but not used here */
border-collapse: collapse; /* collapse border into single line */
border: 1px solid black; /* use border because color same as some content */
left: 0px; /* get rid of small blank space on the left side */
min-width: 500px; /* too crammed to see below this; else scrolbars don't appear */
width: 100%; /* span full screen horizontally */
height: 1em; /* 3 = alt for url popup overlays in dumb browsers (but too big) */
}
.footertable td { /* for all <td> nested in a class=footertable */
padding-left: 5px; /* using both clips rightmost item iff table-layout:fixed */
padding-right: 5px;
}
/* scale up toolbar links for smaller mobile devices */
@media screen and (max-device-width: 640px) {
.footertable {
min-width: 800px; /* need more whitespace for larger font */
height: 1.25em;
}
.blocklinkbar {
font-size: 1.25em; /* go large, for touch */
}
.blocklinkbar:hover {
text-decoration: none; /* else underline/italic may get stuck on scroll */
font-style: normal;
}
}
</STYLE>
<!--
---------------------------------------------------------------------------------------
oddly, outline doesn't appear in IE<=9 only (in this and mergeall-android docs only);
these overwrite/extend base properties above, ignored as a comment by other browsers;
---------------------------------------------------------------------------------------
-->
<!--[if lte IE 9]>
<STYLE>
PRE {
outline: none;
border: 1px solid black; /* order doesn't matter in either */
}
.notebox {
outline: none;
border: 1px solid black;
}
</STYLE>
<![endif]--> <!-- all one big comment ouside IE 5..9 -->
<!--COPIED GENHTML INSERT=============================================================
Nov19: use a floating 'top' button is JS enabled, else keep in toolbar as before.
Copied from main website's genhtml insert: see e.g., python-changes-2014-plus.html.
Does not shade button on hovers because hovers are largely broken on mobiles.
Was usable in toolbar too, but that should generally be for site nav, not page nav.
======================================================================================-->
<STYLE>
#tocBtn {
display: none; /* initially hidden (change on scroll) */
position: fixed; /* float/persist */
bottom: 36px; /* above site toolbar */
right: 8px; /* left of scroll area */
z-index: 99; /* covering priority */
font-size: 15px;
border: none;
outline: none;
cursor: pointer; /* change on mouse-over */
padding: 10px;
border-radius: 4px; /* rounded corners */
color: white; /* foreground (text color) */
background-color: #999; /* which is #999999, which is rgb(153, 153, 153) */
}
/*
Transparency? - pass: obscures button's text always, and obscures page text whenever
visible below button; better to fully obscure and let user scroll to read normally.
opacity: 0.6; ---no: less portable (IE) & blurs button text
background: rgba(153, 153, 153, 0.6); ---punt: obscures both button & text below it
*/
.finalparagraph { /* extra space above Top bottom at end of page */
margin-bottom: 50px; /* else text at end may be overlayed and unviewable */
}
</STYLE>
<!--END INSERT========================================================================-->
<!-- originally:
<TITLE>Mergeall on Android: The Termux How-To</TITLE>
-->
<TITLE>Using Mergeall on Android with Termux (and Pydroid 3)</TITLE>
</HEAD>
<!-- =============================================================================== -->
<BODY>
<!--COPIED GENHTML INSERT=============================================================
Nov19: build+configure a floating to-index button, used iff JS enabled (else a link).
If JS is not enabled, a <noscript> block adds prior static buttons in toolbar instead.
Copied from main website's genhtml insert: see e.g., python-changes-2014-plus.html.
======================================================================================-->
<!-- Button: "display: none" in CSS makes invisible initially, JS or not -->
<!-- this originally used inline onclick="window.location.href = '#TOC';"-->
<button onclick="tocClick();"
id="tocBtn"
title="Go to index">Top</button>
<!-- Set up click/scroll callbacks iff JS is enabled -->
<script>
// Get and save the button in global (page) scope
var tocButton = document.getElementById("tocBtn"); // i.e., "Top" goes to toc
function tocClick() {
//
// Go to the toc, clear hover shading? (else recessed color gets stuck on mobile);
// unhover doesn't work as tried here: see the CSS code in <head> for more details;
//
window.location.href = '#TOC'; // go/scroll to toc tag
//tocButton.style.backgroundColor = '#999'; // no: unhover for mobile?
}
function scrollFunction() {
//
// Adapted from a w3 example; the odd || test is for browser interoperability:
// see, for instance, https://dev.opera.com/articles/fixing-the-scrolltop-bug/
//
var showat = 500;
// N pixels from the top after scroll?
if (document.body.scrollTop > showat || document.documentElement.scrollTop > showat) {
tocButton.style.display = "block"; // show Top
} else {
tocButton.style.display = "none"; // hide Top
}
}
// Show/hide button whenever user scrolls below/above N px from top
window.onscroll = scrollFunction; // source had "function() {scrollFunction()};"; why?
</script>
<!--END INSERT========================================================================-->
<H1>Using Mergeall on Android with Termux (and Pydroid 3)</H2>
<!-- +++++++++++++++++++++++++++PPUS announce+++++++++++++++++++++++++++++++++ -->
<STYLE>
DIV.ppusannounce {
margin: 50px 50px 50px 50px; /* t, r, b, l */
border: thin solid black;
border-radius: 8px;
background-color: #0f2f2f;
color: white;
padding: 8px;
}
DIV.ppusannounce>P:first-child {margin-top: 4px;}
DIV.ppusannounce>P:last-child {margin-bottom: 4px;}
DIV.ppusannounce A {color: cyan; text-decoration: none;}
DIV.ppusdetails {
display: none; /* not inline-block: don't show if no javascript */
margin-top: 0px;
margin-bottom: 0px;
}
#ppusdetailsbtn { /* fixed button to show/hide list on clicks iff JS */
display: block; /* always show: require javascript for spacing/wording */
background-color: #ddd;
color: black;
font-weight: bold;
cursor: pointer;
padding: 6px;
border-radius: 8px;
border: thin black solid;
margin-bottom: 0px;
}
.trimbottom1 {margin-bottom: 0px;} /* manual for dynamic hide/show bits */
.trimbottom2 {margin-bottom: 4px;}
</STYLE>
<DIV class=ppusannounce>
<P>
Preface: as of May 2023, the system described on this page has been
superseded by the newer <I><B>PC-Phone USB Sync</B></I>—a standalone
backup/sync app that runs on both Android 8-and-later
<A HREF="https://quixotely.com/PC-Phone%20USB%20Sync/+screenshots/Android/Tablet-Android13/_thumbspage/t5-sync-action-run.jpg.html">devices</A>,
as well as Windows, macOS, and Linux
<A HREF="https://quixotely.com/PC-Phone%20USB%20Sync/+screenshots/macOS/_thumbspage/08-main-tab.png.html">PCs</A>.
Read about and fetch the new app at its separate
<A HREF="https://quixotely.com/PC-Phone%20USB%20Sync/index.html">website</A>,
and tap the button below to learn how this app improves on prior solutions.
<P class=trimbottom1>
<button id=ppusdetailsbtn
onclick="showppusdetails();"
title="Show/hide details">Details</button>
<DIV class=ppusdetails id=ppusdetailsdiv>
<P>
On Android 11 and later, this new app uses the All files access
<A HREF="https://learning-python.com/mergeall-android11-updates.html#allfilesaccess">permission</A>
to regain access to both shared storage and removable (e.g., USB)
drives. This circumvents the loss of file-path/POSIX access to USB drives
in later
<A HREF="https://learning-python.com/mergeall-android11-updates.html#asb4">Androids</A>.
Shared storage is still slower in recent
<A HREF="https://learning-python.com/mergeall-android11-updates.html#asb5">Androids</A>,
but the new app supports content hosting in faster app-specific
<A HREF="https://quixotely.com/PC-Phone%20USB%20Sync/Usage-Overview.html#Storage">storage</A>,
and UFS 4.0 chips promise to speed storage in
<A HREF="https://duckduckgo.com/?q=ufs+4+storage">general</A>.
As a self-contained app, the new program also eliminates the Termux and Pydroid 3 app
dependencies of this and other former
<A HREF="https://learning-python.com/android-deltas-sync/_README.html">solutions</A>,
and provides a much more seamless and graphical Android
<A HREF="https://quixotely.com/PC-Phone%20USB%20Sync/+screenshots/Android/index.html">experience</A>.
<P>
On PCs, the new program is delivered as a standalone app or executable to avoid extra
installs and mesh with the host platform's paradigms.
And on every platform, this app is still an all-Python system—despite Android's
Java biases. It embeds and uses the portable
<A HREF="https://learning-python.com/mergeall.html">Mergeall</A> system to process content;
codes its GUI in Python with the portable
<A HREF="https://kivy.org/">Kivy</A> toolkit;
builds apps and executables with the
<A HREF="https://github.com/kivy/buildozer">buildozer</A> and
<A HREF="https://pyinstaller.org">PyInstaller</A> tools;
and talks to Java with
<A HREF="https://github.com/kivy/pyjnius">pyjnius</A> only when it must.
In fact, this is still largely Mergeall, with a new and improved GUI launcher
that also provides access to USB drives on recent Androids.
<P class=trimbottom2>
Like all the earlier Android/PC content-sync solutions worked out at
this site over the last five years, the new app is also private by
design: its use of USB drives for folder syncs both boosts speed,
and ensures that your content
remains your property—not someone else's point of control.
This app is available as full and trial versions on the Play store
for Android, and as free downloads for PCs.
Please
visit the new app's
<A HREF="https://quixotely.com/PC-Phone%20USB%20Sync/index.html">website</A>
for more screenshots, more info, and download links.
</DIV>
</DIV>
<SCRIPT>
var ppusdetailsbtn = document.getElementById('ppusdetailsbtn');
var ppusdetails = document.getElementById('ppusdetailsdiv');
function showppusdetails() {
if (ppusdetails.style.display == "inline-block") {
ppusdetails.style.display = "none"; // hide details
}
else {
ppusdetails.style.display = "inline-block"; // show details
}
}
</SCRIPT>
<!-- +++++++++++++++++++++++++++END PPUS announce+++++++++++++++++++++++++++++ -->
<P>
<P>
<TABLE>
<TR><TD align=left valign=top>Version:
<TD>June 18, 2019 (revised Nov-2022)
<TR><TD align=left valign=top>Author:
<TD>© M. Lutz,
<A HREF="https://learning-python.com">learning-python.com</A>
<TR><TD align=left valign=top>Quick links:
<TD><A HREF=".">View</A>
and <A HREF="https://learning-python.com/mergeall-android-scripts.zip">fetch</A> this package<BR>
<TR><TD><TD>
<A HREF="_readme-items/index.html">View</A> this guide's media<BR>
<TR><TD><TD>
<A HREF="https://learning-python.com/using-tkinter-programs-on-android.html">View</A>
the related tkinter doc
</TABLE>
<P>
This document describes how to use
<A href="https://learning-python.com/mergeall.html">Mergeall</A> to sync content
on Android devices. It primarily focuses on running command lines in the
<A HREF="https://termux.com/">Termux</A> app, and more briefly explores
a Mergeall GUI <A HREF="#toc8">alternative</A> on this platform.
This coverage is provided freely, but with no warranties of any kind.
Findings here reflect testing on Samsung Galaxy devices running Android
7, 8, and 9 (Nougat, Oreo, and Pie) and 10. Given the wide
variety of products and rapid—really, supersonic—rate of change in the
Android world, applicability to other devices and Androids cannot be
<A HREF="#toc9">guaranteed</A>.
Mergeall itself changes content by design: see its
<A HREF="https://learning-python.com/mergeall-products/unzipped/UserGuide.html#warn">usage cautions</A>
before using the approaches outlined here.
<H4><A name="TOC">Contents:</A></H4>
<UL>
<LI class=tocmain><A HREF="#toc1">Overview</A>
<LI class=tocmain><A HREF="#toc2">Mergeall Requirements on Android</A>
<UL>
<LI class=tocnest><A HREF="#toc21">Use Android 8 (Oreo) and Higher Only</A>
<LI class=tocnest><A HREF="#toc22">Use FAT32 External Drives on Some Devices</A>
<LI class=tocnest><A HREF="#toc23">Use Mergeall Command Lines in the Termux App</A>
<LI class=tocnest><A HREF="#toc24">Grant Termux Extra Storage Permissions</A>
<LI class=tocnest><A HREF="#toc25">Nest External-Drive Content in Termux App Folders</A>
<LI class=tocnest><A HREF="#toc26">Retain Your Nested Content if Termux is Uninstalled</A>
</UL>
<LI class=tocmain><A HREF="#toc3">Setting up Your Phone and Drives</A>
<UL>
<LI class=tocnest><A HREF="#toc31">1. Install Termux</A>
<LI class=tocnest><A HREF="#toc32">2. Run termux-setup-storage</A>
<LI class=tocnest><A HREF="#toc33">3. Install Python</A>
<LI class=tocnest><A HREF="#toc34">4. Make Your Working Folder</A>
<LI class=tocnest><A HREF="#toc35">5. Install This Package and Mergeall</A>
<LI class=tocnest><A HREF="#toc36">6. Prepare Your Drives</A>
<LI class=tocnest><A HREF="#toc37">7. Edit Your Bash Profile</A>
</UL>
<LI class=tocmain><A HREF="#toc4">Managing Your Content</A>
<UL>
<LI class=tocnest><A HREF="#toc41">Running Syncs and Diffs</A>
<LI class=tocnest><A HREF="#toc41b">The FAT Times Fixer Tool</A>
<LI class=tocnest><A HREF="#toc42">Other Mergeall Tools and Techniques</A>
</UL>
<LI class=tocmain><A HREF="#toc5">Random Tips</A>
<UL>
<LI class=tocnest><A HREF="#toc51">Termux Tips</A>
<LI class=tocnest><A HREF="#toc52">Other Tips</A>
</UL>
<LI class=tocmain><A HREF="#toc6">The Mergeall Termux Wrap-Up</A>
<LI class=tocmain><A HREF="#toc7">Appendix A: A Brief Primer on Android Pathnames</A>
<LI class=tocmain><A HREF="#toc8">Appendix B: The Pydroid 3 tkinter GUI Alternative</A>
<UL>
<LI class=tocnest><A HREF="#toc81">Mergeall's GUI in Action</A>
<LI class=tocnest><A HREF="#toc82">Mergeall Pydroid 3 Requirements</A>
<LI class=tocnest><A HREF="#toc83">Required Mergeall Code Changes</A>
<LI class=tocnest><A HREF="#toc84">How to Run Syncs in Mergeall's GUI</A>
<LI class=tocnest><A HREF="#toc85">Glitches in Pydroid 3 tkinter GUIs</A>
<LI class=tocnest><A HREF="#toc86">The Advertising Scourge</A>
<LI class=tocnest><A HREF="#toc87">General Requirements Still Apply</A>
<LI class=tocnest><A HREF="#toc88">The Mergeall Pydroid 3 Wrap-Up</A>
</UL>
<LI class=tocmain><A HREF="#toc9">Appendix C: Android 11—the Future is Closed?</A>
<UL>
<LI class=tocnest><A HREF="#toc91">The Permissions Game Changer</A>
<LI class=tocnest><A HREF="#toc92">The Impact on Mergeall</A>
<LI class=tocnest><A HREF="#toc93">The Impact on Android</A>
<LI class=tocnest><A HREF="#toc94">Updates: The Rest of the Story</A>
</UL>
</UL>
<DIV class=notebox>
<A name=android11plus>
<P class=noteboxheader>
Android 11+ Update and Alternative
</A>
<P>
<I>Short story:</I> most devices that run Android 11 and later
can no longer use the tools described here, but can use a
newer and equivalent alternative.
<P>
Both the script and GUI solutions described in this doc require
Android Oreo or later, and a device with access to either a USB
drive or a microSD card for syncs. Because Android 11 revoked USB
access for POSIX programs like Mergeall, users of devices which
lack a microSD card and run Android 11 or later should
instead use the <I>Android Deltas Sync</I> alternative solution available
<A HREF="https://learning-python.com/android-deltas-sync/">here</A> and described
<A HREF="https://learning-python.com/android-deltas-sync/_README.html">here</A>.
<P class=noteboxlast>
This alternative syncs content from PC to phone with delta sets
computed from a proxy drive instead of direct comparisons,
but works equally well, and requires roughly the same amount
of user interaction. For more on its rationale, tap
<A HREF="https://learning-python.com/fold3-vs-deltas.html">here</A>. For more on
Android 11's changes, click
<A HREF="https://learning-python.com/mergeall-android11-updates.html#asb4">here</A>.
microSD cards may still be accessible to tools here on Android 11,
but are increasingly rare.
</P>
<!-- original (incremental and not strong enough)...
Android 11 Update
<P>
Regrettably, initial testing of Android 11 in early January 2021
suggests that Mergeall and some of the other programs noted in this guide may be partly
or wholly unusable on this Android today, though this story may improve over time
with app and vendor fixes. Please see the off-page
<A HREF="https://learning-python.com/mergeall-android11-updates.html#android11strikesback">update here</A>
for details, and consider postponing an Android 11 install on your phone
until its full implications for the programs you use are clearer.
<P class=noteboxlast>
<I><B>New:</B></I> a Mergeall work-around has been developed for Android 11's
removal of direct USB access from POSIX programs coded in Python. See the
work-around's
<A HREF="../android-deltas-sync/">package</A>, its usage
<A HREF="../android-deltas-sync/_README.html">guide</A>, and its initial
<A HREF="../fold3-vs-deltas.html">rationale</A>. This work-around may subsume
this page's script and GUI solutions on Android 11 and later.
</P>
-->
</DIV>
<!-- ============================================================================ -->
<H2><A name=toc1>Overview</A></H2>
<P>
If you're able to meet its requirements below,
<I><A href="https://learning-python.com/mergeall.html">Mergeall</A></I> on Android
provides incremental backups and changes propagation for content stored on your smartphone.
With a few simple commands (or a fledgling GUI), you can sync content on your phone's
internal storage or removable SD card both to and from an attached USB drive, and can
do so right on your phone itself. The net effect turns your phone into a portable storage
device for arbitrarily large content collections, without requiring card removal
or device rooting, and without the cost, speed, and privacy negatives of cloud storage.
<P>
The
<A HREF=".">package</A>
that hosts the guide you are reading is a supplement to the Mergeall system.
It comes with precoded <I>helper scripts</I>
that you can run directly on your phone from the Termux app's prompt. Though not
required, these scripts are designed to simplify Mergeall's command-lines mode on
Android, and minimize the risk of user errors. To make edits on smartphones easier,
these scripts also use common settings in the included <code>.bash_profile</code> file,
which you'll modify for your drives' IDs and folder names. Besides their role in
scripts, these settings provide quick command-line access to your folders in general.
<P>
For Mergeall users who break out in hives at the thought of using command lines,
this guide's <A HREF="#toc8">Appendix B</A> also includes coverage of using Mergeall's
GUI in the Pydroid 3 app, and this package includes patch files needed to make the GUI work.
While this approach comes with code changes,
operational issues, and pay-or-else advertising that make it a secondary choice here,
some users may prefer a GUI even if it entails extra drama. If you wish to explore
this option, be sure to read the Termux coverage in the main part of this guide first,
as most of it applies to Pydroid 3 GUIs too.
<DIV class=notebox>
<P class=noteboxheader>
Terminology Notes
<P>
Android's history and variability make documentation challenging.
In this guide and its scripts, the terms "SD" and "USB" are
used to mean a removable microSD card, and anything that works on
the USB port, respectively, and the terms "external" and "removable"
are used interchangeably for a drive that can be detached from
your phone (i.e., SD or USB).
Your device also has nonremovable storage called "internal" here,
though Android often refers to this as "external" and makes it
available in a folder named <code>/sdcard</code>. And if you don't
have a removable card, your phone instead has a nonremovable
"external" partition on your phone's internal drive set aside
for storage—and confusion.
<P>
In the name of generality (and sanity), this package's scripts use the
more function-related "data" to mean the content copy kept on your phone
(e.g., on internal storage or removable SD card), and "plug" to refer
to the content copy on your attached USB drive (e.g., on flashdrive
or portable SSD); extrapolate as needed for your use cases.
The term "phone" crops up informally here too, but this guide applies to
Android devices of all types.
<P class=noteboxlast>
And to any grammarians in this document's audience: some of the closed
compounds used here, including "logfile," "zipfile," and "fullscreen"
may rankle a few purists, but they just make sense (the compounds,
that is, not the purists).
</P>
</DIV>
<!-- ============================================================================ -->
<H2><A name=toc2>Mergeall Requirements on Android</A></H2>
<P>
The <A href="https://learning-python.com/mergeall.html">Mergeall</A>
system, originally developed for PCs, can be used unchanged on Android to
synchronize on-phone content to and from a USB drive, without rooting your phone, and
without requiring SD card removals—an arguably amazing feat in such a limited
form factor. To make this work, though, you'll have to meet a few requirements up front.
In short, Android has multiple issues and permission rules that require a custom
approach; to use Mergeall on this platform, you:
<UL>
<LI>Must use <A HREF="#toc21">Android 8 (Oreo) and higher</A>—due to a former Android timestamps bug
<LI>Might use <A HREF="#toc22">FAT32 on removable drives</A>—due to an Android exFAT interoperability bug
<LI>Might run Mergeall from <A HREF="#toc23">command lines in the Termux app</A>—because its GUI is marginal
<LI>Must grant Termux <A HREF="#toc24">extra storage permissions</A>—else storage options are limited
<LI>Must <A HREF="#toc25">nest removable-drive content</A> in Termux app folders—else it cannot be updated
<LI>Should <A HREF="#toc26">retain any nested content</A> if Termux is uninstalled—else it will be auto-deleted
</UL>
<P>
The sections that follow go into these requirements in more detail. Later sections
cover setup for these requirements, and usage of this package's helper scripts.
The positive take here is that Mergeall on Android may be constrained, but it
does work. While some users may prefer pulling their SD card and syncing on a
PC, this is delicate work in the extreme, and isn't possible on all devices.
With a USB drive, a little up-front work, and either simple command lines or a
GUI, Mergeall brings a more PC-like experience to your on-phone content management.
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc21>Use Android 8 (Oreo) and Higher Only</A></H3>
<P>
The first requirement is an absolute:
Mergeall works on Android devices, but only if they are running 2017's Android 8
(a.k.a. Oreo, and sometimes just
<A HREF="https://en.wikipedia.org/wiki/Android_version_history">O</A>) or later.
Earlier Androids have a showstopper bug which renders content-sync tools like Mergeall
unusable. You can still use Mergeall to manage your phone's content prior to Oreo,
but not on Android itself: remove your card and sync it on your PC.
<P>
The short story on the bug is that earlier Androids are unable to copy
file modification times to either FAT32 and exFAT drives or internal storage on
non-rooted phones: file <I>content</I> can be copied correctly, but copied files
are always stamped with the <I>time</I> at which the copy was made, not that of the source.
This is lethal to content-sync tools like Mergeall, which rely on file timestamps for
change-detection speed. Without timestamp copies, on-phone content copies cannot be compared
to other copies accurately, and quick content syncs are impossible.
<P>
This issue is well-known, long-lived, device-wide, and not specific to
Mergeall, Python, or Termux. In fact, a shell <code>cp -p</code> copy
command fails to copy times with an error before Oreo too—see the Nougat
<A HREF="_readme-items/LOGS/android-timestamps-bug-preoreo.txt">logfile</A>.
It stems from Android's former FUSE-based filesystem, which required root
access to update file times but not content, and was eventually replaced
with Samsung's SDCardFS. For more background, try
the bug report
<A HREF="https://issuetracker.google.com/issues/36940415">here</A>,
the thread
<A HREF="https://forum.xda-developers.com/android/general/guide-timestamp-attributes-correct-t2960935">here</A>,
and searches like
<A HREF="https://duckduckgo.com/?q=Android+setLastModified()+fails">this</A> and
<A HREF="https://duckduckgo.com/?q=Android SDCardFS">this</A>.
As an example of its wide swath, this bug also kills calendar-file backup copies
in the Frigcal program, which makes a cameo appearance in
<A HREF="#pydroidglitches">Appendix B</A>;
prior to Oreo, many programs were impossible on smartphones.
<P>
Though likely unintentional, it's unfortunate that the world's most-used mobile operating
<A HREF="https://www.idc.com/promo/smartphone-market-share/os">system</A>
didn't support non-cloud content syncing until 2017 (and still
doesn't for many prior-version
<A HREF="https://duckduckgo.com/?q=android+version+usage">users</A>).
The good news is that this bug is now fixed, making cloud-free storage on
Android finally practical for non-trivial content—and just in time
for the larger-storage devices beginning to appear.
</P>
<DIV class=notebox>
<P class=noteboxheader>
The Latest Things
<P class=noteboxlast>
<SPAN class=update>Update</SPAN>:
in mid-2020, Mergeall has also been verified to run on <B>Android 10</B>
(formerly called Q), using the latest Termux and Pydroid 3 apps and their
latest Pythons—specifically: Termux 0.99, Pydroid 3 4.0, and Python 3.8 in both.
This means you can run its command lines and GUI on Androids Oreo (8), Pie (9), and 10.
Android 11 (formerly R) support is still murky, given its planned change to internal-storage
<A HREF="#toc9">permissions</A>; Mergeall will likely work on 11, but
content may have to be nested barring app work-arounds.
</P>
</DIV>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc22>Use FAT32 External Drives on Some Devices</A></H3>
<P class=preface>
<SPAN class=preface>Preface</SPAN>:
This section documents an exFAT timestamps issue that exists in former Samsung
Androids. The issue <B>has been fixed</B> as of 2020's
Android 10 per the <A HREF="#exfatepilog">epilog</A>, but will still be present on
older unupdated devices. Hence, users of devices running Android 9 (Pie) and earlier are
still advised to use FAT32 per this section, but users of newer or upgraded Samsung
devices running Android 10 or higher may freely employ exFAT for external drives
hosting Mergeall-managed content—and can safely skip both DST-rollover
issues and this section.
<H4>The Issue</H4>
<P>
Due to a current Samsung Android bug regarding modification times recorded
by the exFAT filesystem, removable drives used with Mergeall on Android
should generally use FAT32 instead of exFAT for interoperability with other
platforms. In short, the modification
times of files created on exFAT drives are known to be incorrectly skewed
as follows when using Android Nougat and Oreo (and later Pie) on multiple
Samsung devices:
<UL>
<LI>Times created on Mac OS
are skewed 16 hours <I>ahead</I> when used on Android
<LI>
Times created on Android
may be skewed 16 hours <I>behind</I> when used on Mac OS and Windows
</UL>
<P>
Both skews were observed in and are relative to the US Pacific time zone,
which is normally 8 hours behind the
<A HREF="https://en.wikipedia.org/wiki/Pacific_Time_Zone">UTC base</A>.
These are likely symptoms of the same exFAT bug in Samsung Android,
and may eventually be fixed in its future versions, but they can wreak
havoc with Mergeall file comparisons today.
<P>
It's important to note up front that, due to testing constraints, it's unknown
whether this flaw lies only in Samsung software or is more widespread.
exFAT is not part of Android's AOSP core-functionality stack
(described
<A HREF="https://en.wikipedia.org/wiki/Android_(operating_system)#AOSP">here</A> and
<A HREF="https://source.android.com/">here</A>) which all phone makers augment
with code from various sources
(described <A HREF="https://en.wikipedia.org/wiki/ExFAT#Reimplementations">here</A> and
<A HREF="https://www.microsoft.com/en-us/legal/intellectualproperty/mtl/exfat-licensing.aspx">here</A>),
and the issue is labeled "Android" in this document for
brevity alone.
Before making filesystem choices, you'll want to compare exFAT-drive
file times on your own PC and Android devices to see if this defect impacts you.
Even if limited to Samsung devices, though, this bug's scope is
<A HREF="https://www.idc.com/promo/smartphone-market-share/vendor">large</A>.
<P>
It's also worth noting that you may be able to use exFAT in
some contexts anyhow. It's now supported on most
<A HREF="https://en.wikipedia.org/wiki/ExFAT#Adoption">Androids</A>;
the first bullet above applies only to content created on Mac OS;
this may be a Samsung-only issue;
and exFAT-drive files may in some use cases
squeak by as <I>incorrect-but-good-enough</I> for Mergeall
comparisons because their mistakes are consistent.
Moreover, users who will only ever backup on-phone content to a USB drive
are exempt from exFAT PC interoperability concerns.
Until this bug is fixed, though, it's dire enough to recommend FAT32 as
the more reliable choice for your Mergeall-on-Android drives.
<P>
If you're willing to take that recommendation on faith, feel free
to skip ahead to the next requirement now. For the curious and
skeptical, though, the following traces the bug's
forensics—beginning at the scene of the crime.
<H4>Act 1: Mac OS Content Is Skewed +16 Hours</H4>
<P>
The first symptom of this exFAT bug is easiest to diagnose:
in testing on three recent Samsung devices (a J7, an S8+, and multiple Note9s),
exFAT file timestamps created on some PC platforms are interpreted differently
on Samsung Android than they are on any other platform tested.
That is, the <I>same</I> file on the <I>same</I> exFAT drive reports a different
modification time on Android alone.
<P>
In a comprehensive test conducted in early 2019 in the Pacific time zone:
<UL>
<LI>
Files were created on the same exFAT-formatted USB flashdrive
and microSD card on Mac OS El Capitan, Sierra, and High Sierra;
Windows 7, 8, and 10; and Android Oreo and Nougat.
Linux wasn't included, because its exFAT support is sketchy.
<LI>
All the created files reported modification times identically
and correctly on all these platforms—<I>except</I> the
two Androids.
<LI>
On the two Androids, <I>only</I> the files created on
all three versions of Mac OS reported modification
times incorrectly skewed 16 hours ahead of their correct values.
<LI>
Files created on Windows and Android reported times correctly
(in this test, at least: stay tuned for more on Android times).
</UL>
<P>
In other words, this test's results make the flaw appear to be a
<I>double bug</I>:
the Mac OS PC platform records exFAT timestamps in such a way
that triggers an incorrect interpretation on the Android mobile
platform. Files created on exFAT drives by Mac OS (only) report
incorrect times on Android (only).
<P>
No, <I>really</I>. For a more graphic look at the problem, here is a
folder of files created on Mac OS High Sierra, as it's seen on
<A HREF="_readme-items/EXFAT-BUG/bugtest-MacOS-on-MacOS.png">Mac OS El Capitan</A>,
<A HREF="_readme-items/EXFAT-BUG/bugtest-MacOS-on-Windows.png">Windows 10</A>,
and
<A HREF="_readme-items/EXFAT-BUG/bugtest-MacOS-on-Android.jpg">Android Oreo</A>
("Today" on Mac OS means 1/15/2019, and 1/16 was still tomorrow when
this test was run).
As you can see, the file times are wrong by 16 hours on Android
alone—files created around 8:20 AM report their times near 12:20 AM
the next day.
<P>
Conversely, files created by this test on Android report the same and correct times on
<A HREF="_readme-items/EXFAT-BUG/bugtest-Android-on-MacOS.png">Mac OS</A>,
<A HREF="_readme-items/EXFAT-BUG/bugtest-Android-on-Windows.png">Windows</A>,
and
<A HREF="_readme-items/EXFAT-BUG/bugtest-Android-on-Android.jpg">Android</A>.
That makes this seem a one-way bug: it impacts just exFAT files created on
Mac OS and viewed on Android, and not vice versa. You'd also never
notice it if you stick to PCs (Windows times are correct on
<A HREF="_readme-items/EXFAT-BUG/bugtest-Windows-on-MacOS.png">Mac OS</A>
and vice-<A HREF="_readme-items/EXFAT-BUG/bugtest-MacOS-on-Windows.png">versa</A>),
or limit your scope to just Windows and Android (Windows times are correct
on
<A HREF="_readme-items/EXFAT-BUG/bugtest-Windows-on-Android.jpg">Android</A> and
vice-<A HREF="_readme-items/EXFAT-BUG/bugtest-Android-on-Windows.png">versa</A>).
As the next section will explain, though, the latter deduction falls apart in the
face of fresh evidence.
<H4>Act 2: File Saves May Be Skewed -16 Hours</H4>
<P>
But wait, it gets worse.
The second symptom of Android exFAT anomaly is subtler and took more digging to unearth:
in addition to the prior section's +16-hour Mac OS content skew, Samsung Android
devices may also <I>write</I> file modification times incorrectly on exFAT drives.
In later tests run on the same Samsung Android devices, files saved to internal storage
and FAT32 drives recorded modification times correctly, but <I>some</I> saves to an
exFAT USB drive erroneously recorded times that are interpreted as
<I>16 hours behind</I> the actual save times on every platform
<I>except</I> Samsung Android itself.
<P>
No, <I>really</I>. The facts in this category are bizarre and even appear to be random.
The incorrect times saved by Android are always incorrect on both Mac OS and Windows, but
not all times saved by Android are incorrect (which is why this effect was uncovered later),
and those that are wrong seem to depend on program and/or context (e.g., one program wrote
new files' times correctly but existing files' times skewed—sometimes).
<P>
This makes this skew difficult to trigger, but easy to spot when it appears.
As a brief example, a folder of files on an exFAT USB drive which began like
<A HREF="_readme-items/EXFAT-BUG/act2-android-updates--start.png">this</A>
was updated and extended by assorted programs on
Android around 11:30 AM on 2/27/2019 (again, in the Pacific time zone),
yielding the following scenes on
<A HREF="_readme-items/EXFAT-BUG/act2-android-updates-android.jpg">Android</A>,
<A HREF="_readme-items/EXFAT-BUG/act2-android-updates-windows.png">Windows</A>, and
<A HREF="_readme-items/EXFAT-BUG/act2-android-updates-macos.png">Mac OS</A>;
as you'll notice, <I>some</I> of the Android-written timestamps are wrong
by -16 hours on the latter two.
<P>
That is, it's not just reads, and it's not just Mac OS: the whole conception of time in whatever
software does exFAT on Samsung Android disagrees badly with others. This is really just
the other side of a single bug's coin, and this guide doesn't have time or space to document
it in more detail (and it's really not this document's job to do so in any event).
But there's clearly something quite wrong with exFAT on Samsung Android today.
<H4>The Mergeall Damage</H4>
<P>
Gory details aside, this is another showstopper for timestamp-based programs
like Mergeall: because exFAT files don't process timestamps correctly on Android,
such files are generally unusable across any platform mix that includes Android.
These files' incorrect times may work on Android itself—Mergeall
doesn't care that times are correct, only that they
differ when files are changed (i.e., consistently incorrect
is good enough). However, the time skews can still cause
problems when using exFAT drives in at least three noticeably
devious ways:
<UL>
<LI>
When Android
<A HREF="_readme-items/EXFAT-BUG/usb-fat32_8BB9-v-exfat_5C3D.jpg"><I>copies</I></A> files
from an exFAT drive, it copies their incorrectly skewed times too.
This suffices for later change detection: skewed times copied will
compare correctly to skewed times on exFAT-drives. Unfortunately, this also
makes the mistake permanent: if the copied files are ever copied unchanged
from Android to PC—or the drive hosting such files on Android is ever
used on a PC—the copied files will all be off by +16 hours, register as spurious
<A HREF="_readme-items/EXFAT-BUG/exfat-16-hour-skew.png">differences</A>,
and be recopied unnecessarily.
Though less tangible, the skewed times copied from exFAT may also pose
other challenges if you ever need to manually reconcile copies in the future.
<LI>
When Android
<A HREF="_readme-items/EXFAT-BUG/usb-fat32_8BB9-v-exfat_5C3D.jpg"><I>saves</I></A> files,
it records times correctly on both internal storage and FAT32 drives.
Paradoxically, this can make the exFAT skew more error prone: if you ever save a file
on your phone exactly 16 hours after its latest save recorded on an exFAT drive,
the updated file will not register as a difference from the exFAT drive,
and will not be propagated to your other devices. This seems unlikely
to occur; the file would have to be saved on your phone <I>only</I> 16 hours
(plus or minus 2 seconds) after the save recorded on exFAT, and this
matters only if you will sync changes <I>from</I> your phone.
Despite their rarity, though, lightning strikes do happen, and
"unlikely" shouldn't be an acceptable term when it comes to your content.
<LI>
When Android
<A HREF="_readme-items/EXFAT-BUG/act2-android-updates-windows.png"><I>saves</I></A> files,
it may record times incorrectly on exFAT drives. Such files may suffice for
Mergeall comparisons on your phone itself: their incorrect -16-hour skew written to
the drive will be offset by an incorrect +16-hour skew when read from the drive
(or, equivalently, on-phone usage will ignore the flaws that skew file times on PCs),
and consistently incorrect is again good enough. Still, "good enough" is also a
term best avoided when your content is on the line; in this case, the incorrect
times recorded by Android on such exFAT files will forever disagree with both
your PC and off-phone reality in general, and again may hamper future reconciliations.
</UL>
<P>
Just as insidious, there seems no way to compensate for the Android time
skews automatically. For example, because exFAT drives may contain content
created on multiple platforms, just <I>some</I> of your files, those saved
on Mac OS, will report incorrect times on Android, and programs like Mergeall
cannot tell which differences are valid and which reflect +16-hour Mac OS skew.
The -16-hour write skew will be present on all PC platforms, and its nearly
random nature seems even more impossible to work around.
<P>
Happily, FAT32 drives both interpret and record times correctly, and are thus
immune to this bug. For instance,
here's what the same file saved on Mac OS looks like on
<A HREF="_readme-items/EXFAT-BUG/times-exfat-usb-macos.png">Mac OS</A> and
<A HREF="_readme-items/EXFAT-BUG/times-exfat-usb-windows.png">Windows</A>;
on Android, this same file's time is skewed on an
<A HREF="_readme-items/EXFAT-BUG/times-exfat-usb-android.jpg">exFAT drive</A>,
but correct on a
<A HREF="_readme-items/EXFAT-BUG/times-fat32-msd-android.jpg">FAT32 drive</A>.
This means FAT32 provides a path for using Mergeall on Android.
Less happily, this precludes mixed FAT32/exFAT usage: because only
exFAT drives skew times, their files won't
compare correctly with the same files on FAT32 drives (as for others)
and spurious differences and recopies will ensue.
<!--cut: <A HREF="_readme-items/eexfat-exfat-usb-android-Termux.jpg">shells</A> -->
<H4>The Verdict</H4>
<P>
In sum, this defect renders exFAT generally unusable
on Samsung (and perhaps other) Android devices, and makes it impossible to
recommend exFAT for Mergeall on this platform today.
<P>
Ultimately, this is
a bug in Samsung or Android code that should be fixed. Although file times
may have been broken on Android in the <A HREF="#toc21">past</A>,
exFAT files today still report and record incorrect modification times on the Samsung
Android platform <I>alone</I>, and neither file sources nor program contexts
justify further deferring a repair. Barring a future Samsung or Android fix,
though, your safest bet is to use FAT32 for all the removable drives you will sync
with Mergeall on Android for both accuracy and interoperability.
<P>
This is regrettable, given exFAT's support for larger files and
compatibility with other filesystems' times. Per its well-known
limitations, FAT32 doesn't support files larger than 4G, and you may
need to manually adjust your FAT32 drives' modification times twice a
year to keep them in sync with non-FAT drives after daylight-savings-time
(DST) rollovers. The internal storage in both your PC and phone, for example,
is usually a non-FAT drive whose UTC times clash with FAT's local times.
<P>
See the FAT times fixer tool <A HREF="#fixfattimes">covered ahead</A>
for a simple way to make the adjustment; if you use FAT32, you'll
run this tool at DST rollovers on your intermediate USB drive
(to match your PC and phone), and possibly on a removable SD card
(to match your intermediate drive). You might also run this tool
on Samsung Android to adjust the times it skews on exFAT drives, but
this will then skew them on all <I>other</I>
<A HREF="_readme-items/EXFAT-BUG/FIXED-IN-ANDROID-10/index.html">devices</A> (yes, !),
and requires extra steps too; FAT-32 seems a better fix.
<P>
The upside is that, thanks to Microsoft patent
<A HREF="https://en.wikipedia.org/wiki/ExFAT#Other_implementations">constraints</A>,
FAT32 is still much more widely usable today than exFAT. More importantly,
its timestamps are implemented more consistently and interoperably;
for Mergeall on Android, this is a killer asset.
</P>
<DIV class=notebox>
<P class=noteboxheader>
<A name="exfatepilog">The exFAT Epilog</A>
<P>
After this section was written, the exFAT bug it
describes was verified to still exist on Samsung devices running <I>Android Pie</I> (9),
in addition to Nougat and Oreo.
Despite this longevity, the web is woefully silent about this bug—which is
hardly surprising, given that timestamps couldn't be copied on
Android at all until 2017's <A HREF="#toc21">Oreo</A>, which made the platform unusable
for an entire class of software.
<!--this was moot it was a Samsung exFAT bug), and is now pointless to include...
It's not impossible
that Linux mount options may be involved (see this
<A HREF="https://unix.stackexchange.com/questions/470856/timestamps-of-files-copied-to-usb-drive">thread</A>
for leads), but it is impossible to use—or even explore—them
on non-rooted phones.
-->
One
<A HREF="https://superuser.com/questions/1390151/mac-os-x-exfat-file-modification-time-issue">thread</A>
looks at a seemingly unrelated issue, but hints at pathology in Mac OS's
exFAT implementation in general.
<P>
For its part, the Mergeall development team has dutifully posted this bug on both
<A HREF="https://developer.samsung.com/forum/board/thread/view.do?boardName=SDK&messageId=361740&startId=zzzzz~&searchType=ALL&searchText=exFAT">Samsung</A> and
<A HREF="https://issuetracker.google.com/124356537">Android</A> websites,
as well as in the Samsung+ app (the US equivalent of Samsung Members at the time),
but doesn't presently hold much hope for a fix.
The Android website has already dismissed the bug as an OEM-specific "won't fix"
(with no apparent interest in forwarding it to the market-leading
<A HREF="https://www.idc.com/promo/smartphone-market-share/vendor">OEM</A>);
there's been no substantive reply from Samsung on either its website or its app
in a month and a half;
and most support forums in the smartphone domain seem more likely to suggest
restarting your phone and counting to 10.
<P>
<SPAN class=update>Update</SPAN>:
shortly after the preceding paragraph was written, <B>Samsung replied</B> on
the Samsung Members app in spring 2019 to acknowledge an issue regarding the
valid bit for time-zone offset
not being set in their exFAT implementation
(background details
<A HREF="https://digital-forensics.sans.org/blog/2010/07/19/exfat-file-system-time-zone-concerns/">here</A> and
<A HREF="https://www.sans.org/reading-room/whitepapers/forensics/reverse-engineering-microsoft-exfat-file-system-33274">here</A>).
This makes exFAT files' times
correct when created and used only on a phone, but fails to accommodate creation and
usage on other platforms, and explains this section's peculiar empirical findings.
The reply
also promised to try to improve the implementation, but cautioned that a fix may take
some time due to dependencies on the current policy.
While it's too early to file this under "happy ending," it now seems
reasonable to look forward to a future repair; exFAT drives will be
much more useful on Samsung devices when those drives are interoperable
with PCs, and the computing world at large.
<P class=noteboxlast>
<SPAN class=update>Update</SPAN>:
the Mergeall team is pleased to report that the Samsung exFAT
time-skew bug <B>has been fixed as of Android 10</B>.
Specifically, a Note 10+ upgraded in 2020 to Samsung's Android 10 reports modtimes
on exFAT drives that correctly match those on PCs, and copies them correctly to
both internal and removable Android storage; view the screenshots proof
<A HREF="_readme-items/EXFAT-BUG/FIXED-IN-ANDROID-10/index.html">here</A>.
This means you won't run into time-skew issues when using an exFAT-formatted
drive for Mergeall on Samsung devices running Android 10 and higher, and can
avoid FAT32's 4G-filesize limit and DST-rollover adjustments. That said,
you'll still want to use FAT32 for drives that may be used on older
Samsung Androids—of which only Oreo (8), Pie (9), and 10 can be used for
Mergeall <A HREF="#toc21">at all</A>. Android 11 is a mixed bag; the Samsung
exFAT fix is still in place, but this Android seems certain to hobble Mergeall
in new and exciting <A HREF="#toc9">ways</A>...
</P>
</DIV>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc23><A name=installs>Use Mergeall Command Lines in the Termux App</A></A></H3>
<P>
On Android, Mergeall's Tk-based
<!-- <A HREF="https://learning-python.com/mergeall-products/unzipped/docetc/docimgs/macosx/composite.png">GUI</A> -->
<A HREF="https://learning-python.com/mergeall-products/unzipped/docetc/docimgs/index.html">GUI</A>
works marginally with caveats, but its
<A HREF="https://learning-python.com/mergeall-products/unzipped/UserGuide.html#cmdline">command-line mode</A>
works fully and qualifier free. To use this mode, you simply need to use Mergeall's <I>source code</I>, and type
<I>command lines</I> to launch it. Although there are multiple ways to run Mergeall's Python source code
on Android, the <A HREF="https://termux.com/">Termux</A> app provides a full-featured Linux experience
that seems best suited to the Mergeall task. Neither Termux nor command lines are required for Mergeall
on Android, but they're suggested for most users.
<P>
In more detail, Mergeall's apps and executables for PCs (Mac OS, Windows, and Linux) do not apply
to Android, and its reliance on the Tk library (and Python 3.X's tkinter interface to it) means that
using its GUI's source code on this platform entails major tradeoffs. Although there is
some new support for using Tk in Python programs run on Android today, it comes with
rude advertising, substantial glitches, and extra steps that make it difficult to recommend
for Mergeall syncs. See the notebox ahead for more on this story; it's evolving, but at this
writing Tk may still be a nonstarter for many Mergeall users on Android.
<P>
Luckily, Mergeall is built with a decoupled architecture that allows its main code to be run
separately from its GUI. To use Mergeall's command-line mode on your phone, you'll need to:
<UL>
<LI>Install the <A HREF="https://termux.com/">Termux</A> app for running Mergeall command lines
<LI>Install Python for running Mergeall's source code within Termux
<LI>Install Mergeall's source-code <A href="https://learning-python.com/mergeall.html">package</A>
<LI>Run Mergeall's source code with command lines in Termux to sync your content
</UL>
<P>
We'll walk through all these steps in more detail later in this <A HREF="#toc3">guide</A>.
This is obviously more work than installing and opening a standalone app, but it's the
easiest workable option for using Mergeall on Android today.
On the bright side, this isn't nearly as difficult as it may
sound. This package's helper scripts simplify the required command lines; the necessary
installs are really just a few minutes of easy up-front work;
and Termux provides a complete and general-purpose Linux command-line environment where
Mergeall's source code is fully at home. As we'll learn <A HREF="#toc4">later</A>, a content
sync is as simple as entering this at the Termux prompt:
<PRE>
bash mergeall-android-scripts/mergeall-plug-to-data-update.sh
</PRE>
<P>
This is roughly just a dozen taps, with the tab completion stressed repeatedly
in this <A HREF="#toc35">guide</A> (and can be even faster if saved to a file).
Although Mergeall's GUI comes with major downsides on Android today, its
helper-script commands should still be easy enough to appease even the most
chronically command-line phobic.
<DIV class=notebox>
<P class=noteboxheader>
But If You Really Hate Command Lines
<P class=noteboxlast>
For an alternative approach, check out <A HREF="#toc8">Appendix B</A>'s
coverage of running Mergeall's GUI with the new tkinter support in the Pydroid 3
app's IDE. There, you'll learn more about using Python tkinter GUIs on Android,
and sample a leading alternative to Termux for running Python code on this platform.
Because this alternative comes with significant tradeoffs—and shares most of the
requirements and setup tasks covered here for Termux—it's been relegated to the
supplemental-reading section of this guide. Despite its rough edges, the GUI does work
in Pydroid 3, and you may wind up preferring it (and this author may too); but this
guide's Android topics still apply, and its suggestions are for a user base at large.
</P>
</DIV>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc24>Grant Termux Extra Storage Permissions</A></H3>
<P>
In addition to installing Termux, you must run its utility script
<code>termux-setup-storage</code> to create Termux's writeable app-specific
folders, and allow it to update files in internal storage and access removable
drives more broadly.
See <A HREF="https://wiki.termux.com/wiki/Termux-setup-storage">this page</A>
for more on the Termux setup command. Without it, Termux—and the
Python scripts like Mergeall that it runs—cannot read or update
most of the folders on your phone's internal storage or removable drives.
This makes content-processing programs like Mergeall unusable in Termux
until you run this command.
<H4>The Before Permissions Picture</H4>
<P>
To illustrate the Termux permissions story, let's run some Python code
within the Termux app. To do this yourself you'd need to install Termux
and Python per <A HREF="#toc3">ahead</A>, but there's no reason to do so yet;
this is just a demo. This also assumes some familiarity with pathnames
(jump ahead to this section's notebox if you need more background); assumes
basic Python knowledge (but you'll probably get the gist of this simple code
either way); and uses Termux 0.65 (if your mileage varies in the future, it's
probably due to a new Termux, a new Android, or both).
<P>
Before the Termux setup command is run, your storage options are severely
limited: on non-rooted phones, you can read and update files in the fully
private Termux app folder <code>/data/data/com.termux</code> (the one holding
your <code>~</code> home), but not much more.
In Termux's Python, with internal
storage at <code>/sdcard</code>, a removable drive named
<code>/storage/8BB9-1202</code>, and tracebacks trimmed for space here:
<PRE>
>>> open('/data/data/com.termux/xxx.txt', 'w').write('1234')
4
>>> open('/sdcard/xxx.txt', 'w').write('1234')
PermissionError: [Errno 13] Permission denied: '/sdcard/xxx.txt'
>>>
>>> open('/storage/8BB9-1202/xxx.txt', 'w').write('1234')
PermissionError: [Errno 13] Permission denied: '/storage/8BB9-1202/xxx.txt'
>>>
>>> open('/storage/8BB9-1202/Android/data/com.termux/xxx.txt', 'w').write('1234')
FileNotFoundError: [Errno 2] No such file or directory: '/storage/8BB9-1202/Android/data/com.termux/xxx.txt'
</PRE>
This code shows most writes (i.e., updates) failing, but read accesses at all
these locations work the same way at this point. As you can see, there's not much
accessible before the Termux setup command is run.
<P>
The Termux app-private folder at <code>/data</code> is the standout exception.
It supports changes out of the box, but is largely useless for Mergeall, because this
folder cannot be accessed outside the Termux app on non-rooted devices. This
includes file explorers, photo viewers, and text editor apps, and disqualifies
this folder for most Mergeall roles. Because you really don't want to store your
content in a location that can be accessed by just one app (and a command-line-only
one at that), Termux's scope is very narrow when first installed.
(But see the update ahead for exceptions.)
<P>
Technically, at this point Termux can also read and update its nested
<code>Android/data/com.termux</code> app-specific folders like the one in the last
command above, provided you make them manually on either removable drives or internal
storage first. But you would have to do so in another app or on a PC: because
Termux cannot access the containing folders yet, it cannot build the required
folder paths under them. To do more, we have to move on.
<DIV class=notebox>
<P class=noteboxheader>
File explorers, SAF, and app-private storage
<P class=noteboxlast>
<SPAN class=update>Update</SPAN>:
Termux eventually added support for the Storage Access Framework (SAF),
which allows specially equipped file-explorer apps to access the Termux
app-private home folder from outside the Termux app. This isn't normal
access and subverts Android permissions, but it does allow broader use
of this otherwise private folder. For more details, see the
<A HREF="https://learning-python.com/android-deltas-scripts/_README.html#termux-app-private-SAF">later coverage</A> in <I>Android Deltas Sync</I>.
</DIV>
<H4>The After Permissions Picture</H4>
<P>
After running the Termux setup command, your storage prospects improve
radically. You—and hence Python scripts like Mergeall—have update
access to all of internal storage, plus Termux's nested app-specific folders
on removable drives:
<PRE>
>>> open('/data/data/com.termux/xxx.txt', 'w').write('1234')
4
>>> open('/sdcard/xxx.txt', 'w').write('1234')
4
>>> open('/storage/8BB9-1202/xxx.txt', 'w').write('1234')
PermissionError: [Errno 13] Permission denied: '/storage/8BB9-1202/xxx.txt'
>>>
>>> open('/storage/8BB9-1202/Android/data/com.termux/xxx.txt', 'w').write('1234')
4
</PRE>
<P>
Read access for these folders is essentially the same as the write access shown,
except that the failing folder here can be read without error too. That is,
after augmenting its permissions with <code>termux-setup-storage</code>, Termux,
and the programs like Mergeall that it runs:
<UL>
<LI>Can <I>read</I> and <I>write</I> files anywhere in the app-private folder (<code>/data</code>), though it's not very useful
<LI>Can <I>read</I> and <I>write</I> files anywhere in internal storage (<code>/sdcard</code>), though its space may be limited
<LI>Can <I>read</I> files anywhere on removable media (<code>/storage/xxxx-xxxx</code>), like SD cards and USB drives
<LI>Can <I>write</I> files on removable media only in its own app-specific folder
</UL>
In more-practical terms, this means that Mergeall can <I>compare</I> content in
any readable folder listed above, but can <I>update</I> content only in the writeable
folders. This has major consequences we'll see in the next section. It's also narrower
than the access available in some apps like text editors and file explorers, but Termux
(and, we'll see <A HREF="#toc8">later</A>, Python-specific alternatives) impose some access
limits today.
<P>
As noted, Termux's setup command creates its app-specific folders, like the one
used in the last command run above, but you can create these folders manually too (and we
will in the setup part of this <A HREF="#toc36">guide</A>). Running the setup command
creates these nested app folders if they don't already exist; this ensures that Termux,
and the scripts it runs, have a place to make changes on removable drives.
<H4>More Permissions Examples</H4>
<P>
For a more comprehensive look at accessible paths which we'll skip in this guide,
see:
<UL>
<LI>
This Python
<A HREF="_readme-items/PYDROID3-TKINTER/CODE/_permissions.py">test script</A>,
and its Termux
<A HREF="_readme-items/PYDROID3-TKINTER/permissions-results-termux.png">results</A>
<LI>
This extended Python updates
<A HREF="_readme-items/LOGS/permissions-Python-updates.txt">session</A>
<LI>
This system
<A HREF="_readme-items/LOGS/permissions-usb-sd-internal-updates.txt">session</A>
that proves USB drives follow the same rules as SD cards
<LI>
This before-and-after <code>termux-setup-storage</code> Termux
<A HREF="_readme-items/LOGS/permissions-indepth-before-after.txt">session</A>
</UL>
More important here is the third and failing command run in the last code listing above:
even with enhanced permissions, Termux still cannot access all the folders on your
removable drives—which leads to the next and perhaps strangest requirement.
<DIV class=notebox>
<P class=noteboxheader>
For More on Android Pathnames
<P>
By now, you may be feeling a bit confused
by folder pathnames if you haven't done much with command lines in the past. For a
gentler introduction to the subject, see this guide's <A HREF="#toc7">Appendix A</A> for
extra background that we'll omit here. Like much technical documentation, this
guide has to walk a thin line between alienating newcomers, and boring experts.
<P class=noteboxlast>
<SPAN class=update>Update</SPAN>:
though uncertain at this writing, <B>Android 11</B> may impose
the same access constraints on internal storage that KitKat imposed on removable
media, and invalidate some of this section's coverage in the process.
See <A HREF="#thefutureisclosed">Appendix C</A> for more on the possible
consequences for Mergeall—and every other program that saves shared
data on the Android platform.
</P>
</DIV>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc25><A name=content>Nest External-Drive Content in Termux App Folders</A></A></H3>
<P>
After you run the Termux permissions command of the prior section, Mergeall is able
to both read and change content anywhere in internal storage, and can read content
anywhere on an external (i.e., removable) drive. To allow Mergeall to make <I>changes</I>
on <I>external</I> drives, though, your content must be nested in the drives' Termux app-specific
folders. In Mergeall-speak, this nesting is not necessary if an external drive will only
ever be the <I>from</I> copy in synchronizations, but it is required if the drive will ever play
the role of <I>to</I>.
<P>
In more-concrete terms, the Mergeall content folders that you will only <I>read</I> on your
phone can be located anywhere, but those you will <I>change</I> (i.e., update) must reside at
either of the following Android locations:
<UL>
<LI>
Anywhere in internal storage, known as <code>/sdcard</code>
(and sometimes <code>/storage/emulated/0</code>)
<LI>
Nested in the <code>Android/data/com.termux</code> folder of your external drives'
<code>/storage/xxxx-xxxx</code> roots
</UL>
<P>
You can use the first of these for on-phone content copies if you have enough space,
but must use the second for content on SD cards and USB drives that you wish to update.
Since this is an arguably bizarre constraint, let's take a closer look at its rules.
<H4>Content in Internal Storage</H4>
<P>
First, the simple case: if you're lucky enough to have a phone with enough
free space to store your content, internal storage is the easiest approach
for on-phone storage, and recommended over SD cards. Internal storage:
<UL>
<LI>Is usually much faster—5X to 10X faster at writing on one device
<LI>Yields paths that are quicker to navigate and type in commands
<LI>Does not require nesting in the Termux (or other) app-specific folder
<LI>Is generally more accessible to other apps per the notebox ahead
<LI>Is immune to the automatic-deletion peril described in the next section
</UL>
<P>
There's more on internal storage's speed for Mergeall-specific tasks
<A HREF="#speedmatters">ahead</A>.
Internal storage is also growing larger in recent phones, and is the only option
for on-phone storage if your device doesn't have a removable card.
To store your Mergeall content in internal storage, simply copy it to a
folder like the following on your phone; <code>/sdcard</code> really means internal
storage, and the <code>MY-STUFF</code> folder name here can be any name you wish:
<PRE>
/sdcard/MY-STUFF
</PRE>
<P>
This seems simple—and even PC-like—but internal storage isn't enough by
itself. For one thing, even if your on-phone content is in internal storage, you'll
still need to sync to and/or from an external USB drive in most use cases. For
another, if your phone's available internal storage space isn't large enough for your
content (and many today are not), you'll need to store your on-phone content on your
device's SD card instead. Since both USB drives and SD cards require nested
storage to support updates, let's turn to this model next.
<H4>Content on Removable Drives</H4>
<P>
Removable storage is generally slower, but it's required if your phone's internal space
is too small, and is still the ultimate in content control.
To store your changeable Mergeall content on external (i.e., removable) storage—including
USB drives and your phone's removable SD card—simply copy it to a
folder nested in the drive's Termux app-specific folder like this;
again, <code>MY-STUFF</code> means your stuff's folder:
<PRE>
/storage/25C9-1405/Android/data/com.termux/MY-STUFF
</PRE>
<P>
Android uses IDs like this path's <code>25C9-1405</code> to uniquely identify
external drives. Your drives' IDs will naturally vary too, and are displayed in popular
Android file explorers (including
<A HREF="https://play.google.com/store/apps/details?id=com.ghisler.android.TotalCommander">this</A>).
<!--
alas, blacklisted by google for ethics violations:
<A HREF="https://play.google.com/store/apps/details?id=com.estrongs.android.pop.pro">
-->
More unusual here: on both SD cards and USB drives, content folders like this path's
<code>MY-STUFF</code> must be nested <I>six-levels deep</I> in the drives' Termux app-specific
folder as shown, if you wish to update them with Mergeall on your phone.
This differs from internal storage, where anything under the <code>/sdcard</code> root works.
<P>
Termux's explanation for why this is so is told on pages
<A HREF="https://wiki.termux.com/wiki/FAQ#Why_Termux_has_read_only_access_to_the_external_SD_card_.3F">here</A> and
<A HREF="https://wiki.termux.com/wiki/Internal_and_external_storage">here</A>.
This restriction on updating removable drives came online in Android KitKat—to the
dismay of both users and <A HREF="https://metactrl.com/docs/sdcard-on-kitkat/">developers</A>—and
has seen multiple later mutations, including app-initiated run-time requests for storage permissions.
Termux, and all other Python apps tested, however, remain limited to their own app-specific folders for
updates to SD cards and USB drives.
<P>
This may seem an odd requirement given that internal storage is wide open to changes,
but it cannot be lifted in the context of Termux command lines today, and isn't
as grievous in practice as it may sound.
For instance, you can still access your content normally in a file explorer (as captured in this
SD-card <A HREF="_readme-items/fileexplorer-data-dir.jpg">screen shot</A>),
and a home-screen shortcut (like this
<A HREF="_readme-items/homescreen-shortcuts.jpg">one</A>)
provides quick access to your nested content folder;
see your file explorer to set one up.
<H4>To Nest or Not to Nest</H4>
<P>
Keep in mind that <I>no nesting is required</I> for content stored in your
phone's internal storage (i.e., in <code>/sdcard</code>), because it's freely writeable.
Moreover, you <I>can</I> store content at the root of USB drives and SD cards
too, but only if you'll only ever <I>read</I>—and never change—it with Mergeall.
Unless you're sure that a removable drive's copy won't be the subject of a future
Mergeall update, it should be nested in the drive's Termux folder.
<P>
Also keep in mind that Android's permission rules have varied much and often
in the past, and seem likely to do so again in the future. While some media may
grow more accessible and some less, it's impossible to foresee tradeoffs
in a platform that seems to change the rules every other Tuesday
(more on this in the following notebox).
<DIV class=notebox>
<P class=noteboxheader>
More on the (In)Accessibility of Removable-Drive Content
<P>
This section noted that internal storage is recommended for storing content on your
phone because it is more accessible to other apps, but this merits a callout here.
Due to the permission rules we met <A HREF="#toc24">earlier</A>, some apps that
do not request broader access, including Termux, are unable to update content on
removable media except in their own app-specific folders. This is why you must
oddly nest your updateable content on removable media, including SD cards
and USB drives, to manage it with Mergeall in this app.
<P>
Unfortunately, this can also make it impossible to update such content in some
other apps. As we'll see in <A HREF="#toc8">Appendix B</A>, the alternative
Pydroid 3 GUI app has the same access limits—on removable media, it can update
only in its own app-specific folder. For such apps, this is a bit of a catch-22:
you must nest content on removable media to update it, but this cuts it off from
other apps' updates. More tangibly, Pydroid 3 cannot update any content nested
in Termux's app-specific folder on removable media—and vice versa.
The net effect is that your choice of updating app is binding, unless you move
your content.
<P>
Luckily, this caveat's scope is limited: it impacts only updates on removable
drives, and does not apply to all apps. You can still freely view nested content on
these drives, and modify it in general-purpose apps like file explorers and text
editors that request broader permissions than Termux and Pydroid 3. If it's important
to also update content in apps with narrower permissions, though, <I>internal storage</I>
is your best option. Especially with the next section's peril factored in, the strange
access limitations of removable drives on Android makes them less well-suited for
single-folder content storage. Watch for a rehash of this topic when we consider
Pydroid 3 at the end of this <A HREF="#pydroidpermissions">guide</A>;
it becomes crucial for alternative apps.
<P class=noteboxlast>
<SPAN class=update>Update</SPAN>:
though uncertain at this writing, <B>Android 11</B> may impose
the same access constraints on internal storage that KitKat imposed on removable
media, and invalidate some of this section's coverage in the process.
See <A HREF="#thefutureisclosed">Appendix C</A> for more on the possible
consequences for Mergeall—and every other program that saves shared
data on the Android platform.
</P>
</DIV>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc26>Retain Your Nested Content if Termux is Uninstalled</A></H3>
<P>
Finally, a bold-font <B><I>caution</I></B>: per normal Android operation (described by Android docs
<A HREF="https://developer.android.com/guide/topics/data/data-storage">here</A> and
<A HREF="https://developer.android.com/training/data-storage/files#WriteExternalStorage">here</A>),
uninstalling Termux also silently
deletes its app-specific folders on all your drives—including
any content folders you've nested there.
Unnested content copies, and unnested working folders in internal storage
are immune to this, but those nested on your SD card and USB drives to
support Mergeall updates may perish with Termux. Because this occurs
without clear warning, it's substantially worse than the prior section's
caveat on removable-media accessibility, and further bolsters the case for
internal storage.
<P>
Specifically: on all devices tested, the <code>Android/data/com.termux</code>
folders on <I>both</I> the removable SD card (at <code>/storage/xxxx-xxxx</code>)
and internal storage (at <code>/sdcard</code>) were automatically deleted when
Termux was uninstalled, along with any and all content placed there by a user.
On one device, a file explorer also attempted to intervene and delete this
content too, but the system apparently beat it to the punch. There is
no reason to nest your content in an app-specific folder on internal storage—and
a root-level folder on internal storage avoids this issue altogether—but
there is no other option for updateable content on removable drives in Termux
(or other Python apps we'll meet in <A HREF="#pydroidpermissions">Appendix B</A>).
<P>
This automatic deletion may sound scary, but it's designed to clear associated space when
you're done using an app, and really isn't much more dangerous than file-explorer deletions
if expected (and an <code>rm</code> command in the Termux shell can be deadlier
still). Before uninstalling Termux, though, be careful to move your nested
content if you wish to retain it. In Android, <I>uninstalls imply deletions</I>.
</P>
<P class=summary>
<B>And that's a wrap</B> on the requirements front.
In exchange for all this, Mergeall provides a free and proven incremental
backup and changes propagation tool for the content you store on Android devices.
Though not for everyone, this Mergeall approach requires neither device
rooting nor card extraction—both of which can be perilous, if not
impossible—and brings robust cloud-free storage management to your phone.
If those benefits offset this section's requirements for your usage—or you
just enjoy a technical challenge in general—read on for more on using
this package's helper scripts to simplify Mergeall runs on Android.
<DIV class=notebox>
<P class=noteboxheader>
Note from a Possible Future
<P>
If a standalone <I>Mergeall app</I> ever
appears, it may obviate the need for both Termux command lines and content nesting
in Termux's app-specific folders, but would still incur Android version and
filesystem constraints. In particular, barring an exFAT fix,
drives would still
be best formatted as FAT32 for <A HREF="#toc22">interoperability</A>.
Given the sloth with which bugs are fixed in commercial products like Samsung
Android, using FAT32 today is likely not a short-term measure. If you're already
using FAT32 drives, the only impact of an app upgrade would be an optional move of
content to unnested drive roots.
<P class=noteboxlast>
<SPAN class=update>Update</SPAN>:
a year later, that possible future has already changed—an
<B>exFAT timestamps fix</B> did
eventually appear in Samsung <A HREF="#exfatepilog">Android 10</A>, which
makes FAT32 formatting unnecessary on devices running this or later versions.
Android 11's filesystem constraints, however, may pose fresh
obstacles for content <A HREF="#toc9">management</A>,
and its latest updates in 2020 may make a standalone Mergeall app
much more <A HREF="#allfilesaccess">crucial</A>.
</P>
</DIV>
<!-- ============================================================================ -->
<H2><A name=toc3>Setting up Your Phone and Drives</A></H2>
<P>
Now that you've seen the what's required to use Mergeall on Android in general,
this section shifts focus to the <I>how</I>. It provides detailed coverage of
steps to follow to prepare your phone and drives for Mergeall use. Here's
a quick rundown of these up-front tasks:
<OL>
<LI class=tocnest>Install <A HREF="#toc31">Termux</A>—to run Mergeall commands
<LI class=tocnest>Run <A HREF="#toc32">termux-setup-storage</A>—to access content
<LI class=tocnest>Install <A HREF="#toc33">Python</A>—to run Mergeall's source code
<LI class=tocnest>Make your <A HREF="#toc34">working folder</A>—to run code and view results
<LI class=tocnest>Install <A HREF="#toc35">this package and Mergeall</A>—to sync your content
<LI class=tocnest>Prepare <A HREF="#toc36">your drives</A>—to add content and meet specs
<LI class=tocnest>Edit your <A HREF="#toc37">bash profile</A>—to set paths used by scripts
</OL>
Most of these are one-time tasks and can be performed in a variety of ways,
but they should generally be done in the order listed here. The section
following this one covers running merges, but assumes the tasks here have
been completed. So grab your phone, PC, and drives, and get ready for
a bit of admin work.
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc31>1. Install Termux</A></H3>
<P>
To be able to run Mergeall's Python scripts, you first need a command-line
environment in which to run them. The Termux app provides a featured-rich
Linux shell for Android that fits the bill; see its
<A HREF="https://termux.com/">homepage</A> for background.
To get started, install the Termux app on your phone from the
<A HREF="https://play.google.com/store/apps/details?id=com.termux">Google Play Store</A>.
Mergeall's command-line mode "just works" in Termux, because Android is
really Linux under the hood.
<DIV class=notebox>
<P class=noteboxheader>
Using Termux Extra Keys (or Not)
<P>
Shortly after this guide was penned, Termux changed its extra-keys row
to be configurable, but in the process also removed keys crucial to
command-line editing (e.g., arrows).
To restore the former keys shown in this guide's Termux screenshots, visit
<A HREF="https://wiki.termux.com/wiki/Touch_Keyboard">this page</A>, and
scroll to its "Extra Keys Row" section. In brief: in Termux itself,
run a Unix <code>mkdir</code> command to create the new folder
<code>~/.termux</code>;
use a Unix editor like <code>vi</code> or <code>nano</code> to make a
file named <code>termux.properties</code> in the new folder and paste
in the line shown on that page; and restart Termux.
<P class=noteboxlast>
<I>Option</I>: some readers may
prefer to disable the Termux row entirely and install an alternative
on-screen keyboard that has all the important bits, like
<A HREF="https://play.google.com/store/apps/details?id=org.pocketworkstation.pckeyboard">Hacker's Keyboard</A>.
<I>Bonus</I>: such a keyboard may also come in handy for tkinter GUIs run
on Android (stay tuned for the details in <A HREF="#toc8">Appendix B</A>).
</P>
</DIV>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc32>2. Run termux-setup-storage</A></H3>
<P>
To allow Termux to update both internal storage and its app-specific folders
on removable drives, launch the Termux app you just installed on your phone,
and run the following command at its prompt (the <code>$</code> in command listings
here is Termux's default prompt; type the text that follows it, and press whatever
means "enter" on your keyboard to submit the command):
<PRE>
$ termux-setup-storage
</PRE>
<P>
After that, allow access in the
<A HREF="_readme-items/termux-setup-storage.jpg">popup</A> that appears.
Note that you may need to rerun this command later with removable drives
connected, if you get permission errors. This command also creates the
required <code>/Android/data/com.termux</code> nested folders on all connected
media automatically if they do not exist, but you can also make these manually
when you're preparing your drives on a PC (as we will a few steps ahead).
<P>
If you've forgotten why this permissions step is required (or came here from
a search), see the earlier <A HREF="#toc24">coverage</A>.
Also note that Termux commands that update your phone's internal storage
from here on in this guide assume that this command has been run; without it,
such commands will fail on permission errors.
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc33>3. Install Python</A></H3>
<P>
Mergeall is coded in the Python programming language. As mentioned
<A HREF="#toc23">earlier</A>,
we need to use its source code on Android because its PC executables don't
apply, and source code requires a Python interpreter on any platform. To
install Python for use in Termux, simply type the following command at the
Termux app's prompt on your phone, and answer "y" (yes) when prompted:
<PRE>
$ pkg install python
</PRE>
<P>
This gets the latest Python 3.X (version 3.7 at this writing), which works
best for Mergeall. If you want to verify the install, type just
<code>python</code> at the Termux prompt:
<PRE>
$ python
Python 3.7.1 (default, Oct 21 2018, 18:20:26)
[Clang 7.0.2 (https://android.googlesource.com/toolchain/clang 003100370607242d on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import sys, os, platform
>>> sys.platform, os.name, platform.machine()
('linux', 'posix', 'aarch64')
</PRE>
<P>
As usual, a <code>Ctrl+d</code> key combination at Python's <code>>>></code>
prompt gets you back to Termux's <code>$</code> prompt.
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc34>4. Make Your Working Folder</A></H3>
<P>
Now that you've got Termux and Python ready to go, it's time to setup
a working folder on your phone where you'll run scripts to sync content.
This is a simple task, but you'll want to decide up front where you wish
to work, to avoid having to repeat steps later.
<P>
First, the recommendation: an <code>/sdcard/work</code> folder located in the
root of your internal-storage space is an ideal candidate for your working folder,
because it's fast, accessible to file explorers and most other apps, changeable
by programs run in Termux, and short to type. This folder has the following multiple
aliases on devices used in testing, but shorter is generally better on a smartphone:
<PRE>
/sdcard/work
/storage/emulated/0/work
</PRE>
<P>
You can work in any writeable folder on your phone, but others come with
tradeoffs that make them less than ideal in this role. For example:
<UL>
<LI>
Your Termux home folder known as <code>~</code> (which is really
<code>files/home</code> in <code>/data/data/com.termux</code>) is writable,
but fully app private: its lack of visibility in file explorers and other apps
on non-rooted phones can make maintenance tasks like Bluetooth transfers harder.
<LI>
The Termux app's nested folder in internal storage
(<code>Android/data/com.termux</code> in <code>/sdcard</code>) might
seem a natural too, but it's more to type, and will be automatically
deleted if Termux is uninstalled (a broad issue discussed <A HREF="#toc26">earlier</A>).
</UL>
<P>
For this guide, we'll assume and use the recommended working folder in internal
storage (and use its shorter <code>/sdcard</code> name), because this is the
simplest and most-functional location. To create your folder, run the
following in Termux on your phone:
<PRE>
$ mkdir /sdcard/work
</PRE>
<P>
You can also create this folder in your favorite Android file explorer—one
of the benefits of locating it in internal storage this way. Once you've got
your folder set to go, you're ready to populate it with software, per the
next step on our list.
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc35><A name=installs>5. Install This Package and Mergeall</A></A></H3>
<P>
Wherever you choose to work, unzip (i.e., extract) <I>both</I> this
helper-scripts package and Mergeall's source code there for convenience.
The helper scripts simply augment Mergeall, so you'll need both packages.
You can use the standard Mergeall source-code package, because its
core syncing code developed for Mac OS, Windows, and Linux works
<I>unchanged</I> on Android—a testament to the benefits of portable
software (though the story for Mergeall's GUI in <A HREF="#toc8">Appendix B</A>
won't be nearly as seamless).
<P>
For readers newer to Unix systems, the following is one way to
install this helper-scripts package on your phone by running commands
in Termux itself. First, in Termux, go to the working folder you
created in the prior step:
<PRE>
$ cd /sdcard/work
</PRE>
Next, fetch (i.e., download) this helper-script package and unzip it to
your new working folder; click the links here on your phone to perform the
fetch part of this recipe, then type the commands in Termux:
<PRE>
<I>Fetch the zipfile at <A HREF="https://learning-python.com/mergeall-android-scripts.zip">learning-python.com/mergeall-android-scripts.zip</A></I>
$ mv ../Download/mergeall-android-scripts.zip .
$ unzip -d . mergeall-android-scripts.zip
</PRE>
<P>
Finally, do similar to install Mergeall's source-code package in the same working folder:
<PRE>
<I>Fetch the zipfile at <A HREF="https://learning-python.com/mergeall-products/Mergeall-source.zip">learning-python.com/mergeall-products/Mergeall-source.zip</A></I>
$ mv ../Download/Mergeall-source.zip .
$ unzip -d . Mergeall-source.zip
</PRE>
<P>
For an example of the resulting working folder, see this
<A HREF="_readme-items/fileexplorer-working-dir.jpg">screen capture</A>.
This assumes that clicking the links above on your phone makes the zipfiles
show up in <code>/sdcard/Download</code>, but that's normal Android public-folder behavior.
<P>
Two tips here. First, if you're new to Unix, be sure to use Termux's
<I>arrow keys</I> to recall prior commands to edit and resubmit, and its
<I>tab completion</I> to let Termux fill in most of these commands for you.
The latter makes command lines much easier to type; for instance, a tab
after "m" in the sessions above likely expands it to "mergeall-android-scripts.zip,"
one after "M" fills in the Mergeall full zipfile's name, and "Dow"+tab expands to
"Download" similarly. Don't type what you don't have to type—especially
on a phone.
<P>
Second, there are other ways to achieve most of the above that we'll skip here.
For instance, you can fetch files on a PC and beam them to your phone; can use
Android file explorers instead of command lines for most steps in this process
(the former often call an unzip an extract, and refer to <code>.</code> as the
current path); and can even download the zipfiles directly in Termux using
<code>curl</code> after a <code>pkg install curl</code>:
<PRE>
$ cd /sdcard/work
$ curl -O https://learning-python.com/mergeall-products/Mergeall-source.zip
$ unzip -d . Mergeall-source.zip
</PRE>
<P>
Termux has the <code>wget</code> download command too.
Regardless of their sizes, Linux-based devices offer an embarrassment of options.
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc36>6. Prepare Your Drives</A></H3>
<P class=preface>
<SPAN class=preface>Preface</SPAN>:
As noted <A HREF="#exfatepilog">earlier</A>,
the requirement to format removable drives as FAT32
has been lifted as of Samsung Android 10,
but still applies to drives used on older versions of that
platform. Given that Android updates are not all mandatory,
this yields unavoidable bifurcation;
this section's parenthetical updates address the schism.
<P>
Depending on the size of your content, the next step may take the longest.
To use your content on Android, its storage structure must satisfy the
media's filesystem and location requirements we met earlier
<A HREF="#toc22">here</A> and <A HREF="#toc25">here</A>.
In brief:
<UL>
<LI>All <I>removable drives</I> to be used with Mergeall on Android should generally be
formatted with the FAT32 filesystem for best interoperability
(till Samsung <A HREF="#exfatepilog">Android 10</A>), and must nest content in
their Termux app-specific folders if you plan to update them with Mergeall. These
requirements apply to both removable SD cards used for on-phone storage, and USB drives used
as intermediaries between phones and PCs. Setup and content copies for both can be done
on your PC—an advantage of removable drives.
<LI>If you opt to store on-phone content on <I>internal storage</I> instead of SD card,
it does not require FAT32 formatting or content nesting, but content copies
must be done on your phone itself, and your USB go-between drive must
still satisfy the prior bullet's rules.
</UL>
<P>
Since this is probably the most convoluted of the steps in this list
(and is also required of the GUI alternative in <A HREF="#toc8">Appendix B</A>),
the following sections provide a more detailed breakdown by media type.
<H4>To Setup Removable SD Cards and USB Drives</H4>
<P>
Use your PC to setup the removable drives you'll use on Android.
First, you'll need to meet the FAT32 requirement
(till Samsung <A HREF="#exfatepilog">Android 10</A>). If your drive is already using
FAT32, you can move on to the next paragraph. Otherwise, start by copying any existing
Android content on your drive to a temporary location so it can be restored later;
this includes existing <code>DCIM</code> (camera) and <code>Android</code> (app-data) folders,
if present. Then, format your drive as FAT32, and restore the prior existing content you
saved. If you're new to formatting, see the notebox ahead for tips.
<P>
Next, place your content-copy folder on your drive. As noted earlier, if you're sure you will
never sync content <I>to</I> a removable drive on Android, this folder can be stored anywhere
on the drive, including its root; copy or move it there using your usual tools. To allow Mergeall
to update the drive's content, though, its content-copy folder must be placed in Termux's
nested app-specific folder, as follows:
<UL>
<LI>
If you <I>did</I> format your drive, create its <code>/Android/data/com.termux</code> folder
now if it does not exist, and copy your content folder there using whatever copy technique
you normally use on your PC (e.g., file explorer or command line).
<LI>
If you did <I>not</I> format your drive, create its <code>/Android/data/com.termux</code>
folder now if needed, and either move your content there if it's already located elsewhere
on the drive, or copy it there anew.
</UL>
<P>
For the last bullet above, you can <I>move</I> your content folder to the Termux app folder
on the same drive with a single command on Unix-like PCs; here's an example on Mac OS (on
Linux, use your media's paths; on Windows use drive letters instead of volume names,
and move with DOS commands):
<PRE>
~$ mv /Volumes/EXT256-1/MY-STUFF /Volumes/EXT256-1/Android/data/com.termux/MY-STUFF
</PRE>
<P>
A caution here: you can move existing content in a file explorer too, but be sure a move in
yours is a true move, and not a copy-and-delete. The speed difference can be staggering
for large folders, and some Android file explorers may miss the mark on this.
<P>
Whether your content was copied or moved, it's now located in the Termux app
folder—and four levels down from the drive's root—which can be inconvenient
when coding paths and browsing for files. To make this easier to access on your PC,
add a <I>link</I> (or shortcut) to it at your drive's root; here's how to do so
on Mac OS with a symlink
(that looks like <A HREF="_readme-items/macos-symlink.png">this</A>):
<PRE>
~$ ln -s /Volumes/EXT256-1/Android/data/com.termux/MY-STUFF /Volumes/EXT256-1/MY-STUFF
</PRE>
The link (or shortcut) is optional, but can save a lot of navigation time
in file explorers, and typing in command lines. The root-level link probably won't
work on Android, and you can't create one for removable drives on non-rooted phones
because you don't have the required permissions on removable drives (this is the whole
reason for nesting content in the first place); but it can save steps on PCs.
<P>
On Android, instead make a home-screen shortcut (that looks like
<A HREF="_readme-items/homescreen-shortcuts.jpg">this</A>)
to your content folder; a longpress
in your file explorer is the usual first step. This makes more sense
for a resident SD card than a transient USB drive; but because your content is
now six levels deep when viewed on Android—nested in the
<code>Android/data/com.termux</code> folder of your drives'
<code>/storage/xxxx-xxxx</code> root—the shortcut can save a
whopping five taps per access, plus a file-explorer open.
<H4>To Setup Internal Storage on Your Phone</H4>
<P>
If you're able (or required) to store your on-phone content in internal storage,
you don't need to reformat as FAT32
(even before Samsung <A HREF="#exfatepilog">Android 10</A>),
or nest your content in the Termux app-specific
folder. In fact, you shouldn't—internal storage is already formatted,
and nested Termux app folders disappear on Termux uninstalls as noted
<A HREF="#toc26">earlier</A>.
Instead, simply create a folder for your content at the root of internal storage,
and copy your content there. Both these steps must be done on your phone itself.
<P>
To begin, create your content folder in an Android file explorer, or use a Unix
<code>mkdir</code> command in Termux itself. For example, the following creates a
content folder named <code>MY-STUFF</code> using a command in Termux; as usual,
your folder name may vary:
<PRE>
$ mkdir /sdcard/MY-STUFF
</PRE>
<P>
Then, copy your content to your phone. For most users, this will employ an
intermediary USB drive: copy from PC to USB drive, then from USB drive to phone.
A content copy on a USB drive you've already setup for Android in the prior
section will suffice, and avoids another step.
<P>
A caution here too: on the phone, you can run the copy to internal storage in most
Android file explorers, but some fail to copy over file modification times.
At least one popular app, for instance, simply stamps every copied file with the
<I>current time</I>, which is a showstopper for Mergeall—and no better
than the pre-Oreo Android <A HREF="#toc21">bug</A> (in fact, this may
be a lingering accommodation of it).
<P>
To make sure times make the move too, use Mergeall's <code>cpall.py</code> bulk-copy
utility on your phone instead. There's more on this utility
<A HREF="#othertools">ahead</A>;
in short, the copy on Termux looks like the following, assuming the settings we will
create in the next and final <A HREF="#toc37">step</A> are in place. That is,
you'll need to return here and run this after setting drive IDs in your bash profile file
(USB drive IDs are not available until that drive is prepped and attached,
and there's no way to show you this command without them):
<PRE>
$ cd $work
$ py Mergeall-source/cpall.py $plug $data -skipcruft -vv -u > Admin-Mergeall/importlog.txt
</PRE>
<P>
Here too, once the content copy finishes, you may wish to create a home-screen
<A HREF="_readme-items/homescreen-shortcuts.jpg">shortcut</A> to its new location
on your phone. In this mode it's only two levels down in internal storage when
viewed on Android—at <code>/sdcard/MY-SYUFF</code>—but the shortcut
still saves one tap per access, plus a file-explorer open.
A symlink doesn't make sense here, because it would be two levels down too (and
may be impossible in any event: a <code>ln -s</code> at Termux's prompt didn't
work for <code>/sdcard</code>, despite having write permissions there).
<P>
A reminder: even if you use internal storage for on-phone content, you'll
also need to setup a USB drive to propagate changes between your PC and phone
(or save on-phone changes in general).
That removable drive will still require FAT32 formatting for interoperability
(till Samsung <A HREF="#exfatepilog">Android 10</A>),
and content nesting for updates as described earlier
(<A HREF="#toc22">here</A> and <A HREF="#toc25">here</A>), and can be used to
initially populate your internal storage with content. Your on-phone content,
though, is substantially easier to setup and use.
<DIV class=notebox>
<P class=noteboxheader>
Tips on Formatting Your Drives
<P>
As noted, you'll generally need to reformat your
removable drives unless they already use FAT32
(or are used only on Samsung <A HREF="#exfatepilog">Android 10</A> or later),
and this isn't a trivial task if you're
new to it. On Windows, the standard file Explorer won't format
larger drives as FAT32 (it forces exFAT instead), but a DOS <code>format</code>
command and third-party options will; for pointers, try
<A HREF="https://duckduckgo.com/?q=windows+format+fat32+large+card&t=ffab&ia=web">this search</A> and
<A HREF="https://superuser.com/questions/1179871/how-to-format-a-disk-sdcard-as-fat32-in-windows-10">this thread</A>.
<P>
On Mac OS, the Disc Utility app's Erase happily formats larger drives as FAT32
(if you select this and use uppercase drive names), and Linux users have many
<A HREF="https://duckduckgo.com/?q=format+usb+drives+as+fat32+on+linux">options</A>.
Android itself can format drives too (see your drive's Format in
Settings ⇨ Device maintenance/care ⇨ Storage ⇨ Storage settings);
unfortunately, it seems to silently choose FAT32 or exFAT for you based on drive
size (a case of user-friendliness gone bad) so you'll have to use a PC for larger drives.
<P class=noteboxlast>
On all platforms, the usual warning about
formatting applies: it erases your drive, so copy off anything valuable first.
You may also want to add a <code>.nomedia</code> file at the top of reformatted
removable drives that will be duplicates of the content kept on your phone
(e.g., on a go-between drive that will be used to propagate changes between your
phone and PC); where it works, this "hidden" file prevents Android's media scanner
from searching the redundant copies.
</P>
</DIV>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc37>7. Edit Your Bash Profile</A></H3>
<P>
Last but not least, you'll now install and edit the bash profile file shipped in
this package. This must happen last, because you may not know the Android names
of your drives until they've been prepared and attached to your phone. Formatting,
if run, changes the IDs Android uses to identify drives, and you'll
need these IDs in this step.
<H4><A name=bashfile>Installing the Profile File</A></A></H4>
<P>
Along with its scripts (and this guide), this helper-scripts package includes a
prototype bash profile file named <code>your--.bash_profile</code>. This file has
simple settings that provide quick access to your Termux home and working folders,
as well as your content-copy directories on your phone and removable drives.
You can view this file's contents <A HREF="your--.bash_profile">here</A>;
it's mostly documentation (the lines that start with a <code>#</code>).
<P>
To use this file, delete the "your--" in its name; edit its path settings as both its
comments and the next section describe; and either save this file in your Termux home
folder or copy-paste its code into a <code>~/.bash_profile</code> you already use.
Because your Termux home folder is app private, you will normally need to install the
profile file there using commands in Termux itself (not a file explorer). Here's the
required Unix incantation; this assumes you're
working in <code>/sdcard/work</code> and have already unzipped this package there:
<PRE>
$ cd ~
$ cp /sdcard/work/mergeall-android-scripts/your--.bash_profile .
$ mv your--.bash_profile .bash_profile
</PRE>
<P>
For example usage of this file's settings at the Termux prompt, see this
Android <A HREF="_readme-items/Termux-home-work-data-dirs.jpg">screen capture</A>
(and zoom in and zoom often).
The bash profile file is also integral to using the helper scripts here: per the
next and final setup section, it defines Android paths in a single location
shared by all.
<H4>Editing Your Path Settings</A></H4>
<P>
To make both commands and edits easier on smartphones, the helper scripts in this
package use common content-copy path variables set in your bash profile file.
This way, paths need be changed in that file only, and only when you first install
this package or wish to use a new drive.
<P>
When you do need to change paths, the required edits are simple: in the
<code>~/.bash_profile</code> you created from the
<A HREF="your--.bash_profile">prototype file</A>, the settings to change are marked
as "EDIT ME." Replace these precoded paths' "xxxx-xxxx" IDs and literal
"MY-STUFF" with your own drive IDs and content-folder name, respectively.
Again, your drive IDs can be found in Android file explorers like
<A HREF="https://play.google.com/store/apps/details?id=com.ghisler.android.TotalCommander">this</A>
<!--
alas, blacklisted by google for ethics violations:
<A HREF="https://play.google.com/store/apps/details?id=com.estrongs.android.pop.pro">
-->
(make sure to match the case of drive ID letters exactly),
and folder name is whatever you wish it to be.
Here's one way to edit the profile file in Termux—because it's not
visible to other apps, you'll generally need to edit it in Termux itself too:
<PRE>
$ cd ~
$ vi .bash_profile
$ source .bash_profile
/sdcard/work
</PRE>
This runs <code>source</code> to load your new settings after you're done,
and uses the Unix <code>vi</code> text editor.
If <code>vi</code> is too cryptic for your tastes, the more
user-friendly and WYSIWYG <A HREF="_readme-items/nano-on-termux.png"><code>nano</code></A>
text editor also works in Termux after a <code>pkg install nano</code>.
You can also rename and edit the bash profile file on your PC and beam it over,
but you'll still need to copy it into your home folder in Termux, because that
folder is visible nowhere else.
<P>
Special cases: if you opt to use internal storage for your on-phone content copy
instead of SD card (per the requirements section <A HREF="#content">earlier</A>),
use the profile file's <code>/sdcard/MY-STUFF</code> data-path setting instead.
You may need to also edit the scripts themselves to configure for your
Mergeall source-code package's install location if it's not in <code>.</code>
(the current working directory), but this is not required if Mergeall's package
is unzipped to the same folder as this scripts <A HREF="#toc35">package</A>.
<P>
Because removable drives are identified by drive IDs in Android, the paths
used by helper scripts will change if a new removable drive is inserted. To reconfigure
for a new drive, simply update paths in your <code>~/.bash_profile</code> with the editing
commands above; all Android helper scripts automatically <code>source</code> (i.e., reload)
this file on startup, so they always use its current settings.
You may also need to rerun Termux's <code>termux-setup-storage</code> when changing
to a new drive; if you get permission errors, this applies to you.
<DIV class=notebox>
<P class=noteboxheader>
Helper-Scripts Coding Notes
<P class=noteboxlast>
The symlinks made at drive roots by the Mac OS move-and-link
<A HREF="_macos-move-and-link.sh">script</A> here might suffice for pathnames in other
scripts, but these links won't generally work on Android, and non-rooted phones don't have
permission to make such links in removable drives on Android itself.
Termux's <code>~/storage/external-*</code> links also record drive
paths, but they seem too ambiguous to use: they give IDs but not drive type, and are
not always current.
Instead, the helper scripts rely on the manual <code>$data</code> and <code>$plug</code>
settings in your bash profile file, both for generality, and because explicit is
usually better than implicit—especially when your content is at stake.
</P>
</DIV>
<!-- ============================================================================ -->
<H2><A name=toc4>Managing Your Content</A></H2>
After you've performed the prior section's setup requirements, you're finally good to go:
whenever you wish to sync and compare your content, simply run the helper scripts
and other command lines described in the sections that follow. There's generally
no reason to repeat setup steps (unless you wish to use a new drive), but
the commands here can be rerun anytime you wish to propagate changes to or
from your phone.
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc41>Running Syncs and Diffs</A></H3>
The helper scripts shipped in this document's <A HREF=".">folder</A> make it
easier to use Mergeall on smartphones, by precoding common use cases, and
utilizing shared path settings. Among the collection, you'll find helper
scripts that run:
<UL>
<LI>The main Mergeall content backup/propagation script, <code>mergeall.py</code>
<LI>Mergeall's <code>diffall.py</code> byte-for-byte comparison utility
<LI>Unix move+link commands to setup your content copies for Mergeall use on Android
</UL>
<P>
Additional scripts run modification time (a.k.a. modtime) adjustment tools for FAT32
drives, and other Mergeall utilities are available as standard Unix command lines.
As a bonus, all the Android helper scripts here automatically save logfile output to
your working folder's <code>Admin-Mergeall</code> subfolder, using filenames either
passed or generated; to see how this works, let's jump right into the available commands.
<H4>Syncing Commands</H4>
<P>
To sync or compare your content: select the Mergeall <A HREF=".">helper script</A>
in this package whose name matches your goal; open the Termux app on your phone;
and run the selected script with a <code>bash <I>scriptname</I>.sh</code> command
line at the Termux prompt. For example, the following launches a USB-to-phone
updates run, assuming this package has been unzipped in your working folder:
<PRE>
$ bash mergeall-android-scripts/mergeall-plug-to-data-update.sh
</PRE>
<P>
The <code>bash</code> command just tells Termux to run all the precoded commands
in the chosen file, and scripts' filenames imply their action. For instance,
<code>plug-to-data</code> in this script's name syncs changes from USB drive to on-phone storage,
and <code>update</code> means apply changes to the destination automatically;
<code>data-to-plug</code> syncs the other way, and <code>report</code> means report
differences only without changing anything.
<P>
Worth reiterating: be sure to use shell tab completion to fill in the folder and script
names in these commands; in this case "bash m+tab/m+tab" is most of the command.
Optional: scripts located in your Termux home folder (<code>~</code>) can
also be made more directly executable with a <code>chmod +x <I>scriptname</I>.sh</code>
and run with just a <code><I>scriptname</I>.sh</code>; in Termux this works for
home-folder scripts only, not for working folders located
<A HREF="https://wiki.termux.com/wiki/FAQ#Why_do_I_keep_getting_.27permission_denied.27_when_trying_to_launch_a_script.3F">elsewhere</A>.
<H4>Command Variations</H4>
<P>
Mergeall's scripts—and hence this package's bash helper scripts that run them—both
process files and generate logfiles that detail their actions and results. On Android, script
<I>logfile output</I> is automatically saved in your working folder, to subfolder
<code>$work/Android-Mergeall</code>, in a file whose name is either:
<UL>
<LI>Passed as an argument on the script's command line—in
<code><I>filename</I></code>, for <code>bash <I>scriptname</I>.sh <I>filename</I></code>
<LI>Generated to be unique with a date-time stamp—in
<code>mergeall-<I>date</I>-<I>time</I>.txt</code>, when
<code><I>filename</I></code> is omitted
</UL>
<P>
Whether the logfile name is passed or generated, the logfiles subfolder is created
automatically if needed.
<P>
You can also launch scripts with <code>&</code> at the end of the bash command
(or use <code>Ctrl+z</code> and <code>bg</code> after it starts) to run in the background,
so you can monitor output with a <code>tail -f <I>logfile</I>.txt</code>. For instance,
the following session ensures that Termux is in your working directory by running the
<code>gowork</code> command defined in <code>~/.bash_profile</code>; starts a USB-to-phone
update run, saving its output using a generated filename; and monitors script progress:
<PRE>
$ gowork
$ bash mergeall-android-scripts/mergeall-plug-to-data-update.sh &
$ tail -f Admin-Mergeall/mergeall-<I>somedate</I>-<I>sometime</I>.txt
<I>output scrolls here</I>
<I>Ctrl+c to exit</I>
</PRE>
<P>
Alternatively, the following reports file differences only, saves output to
an explicitly named file, and views output only after script completion
with the <code>vi</code> text editor (<code>nano</code> also works after
an install as noted earlier, you can switch to another Termux session while
the merge runs, and you don't really need to <code>gowork</code> again unless
you've gone elsewhere):
<PRE>
$ gowork
$ bash mergeall-android-scripts/mergeall-plug-to-data-report.sh ma.txt
$ vi Admin-Mergeall/ma.txt
</PRE>
<H4>The diffall.py Utility</H4>
<P>
Besides its main merge script, Mergeall includes the <code>diffall.py</code>
deep-comparison utility, and the helper scripts include a script to launch it.
The following, for example, compares content copies byte for byte on Android,
monitoring output as it appears in a passed-in filename; there is no
<I>from</I> and <I>to</I> ordering for drives here, because this isn't a merge:
<PRE>
$ gowork
$ bash mergeall-android-scripts/diffall-plug-and-data.sh diffs.txt &
$ tail -f Admin-Mergeall/diffs.txt
<I>output scrolls here</I>
<I>Ctrl+c to exit</I>
</PRE>
<P>
As a guideline, you should run this comparison script occasionally to ensure that all
files in your content copies are completely identical. Though Mergeall's time-and-size
comparisons are fast and adequate for your day-to-day syncs, drives can and do go
bad eventually. A byte-for-byte check proves the integrity of your content—albeit
much more slowly (read the notebox ahead for details).
<H4>Syncs and Diffs in Action</H4>
<P>
For a more graphic look at the Mergeall sync and diff helper scripts in action
on Android, check out the Mergeall-run screenshots
<A HREF="_readme-items/run-mergeall.png">with</A> and
<A HREF="_readme-items/run-mergeall-notail.png">without</A> tails,
and the exemplary but heavily redacted
<A HREF="_readme-items/LOGS/mergeall-20190106-151653.txt">Mergeall</A> and
<A HREF="_readme-items/LOGS/diffall-20190106-130706.txt"><code>diffall.py</code></A>
logfiles.
And for a comprehensive test that verifies the approaches laid out in this document,
study the files in <A HREF="_readme-items/VERIFY-RUNS/">this folder</A>,
especially its <A HREF="_readme-items/VERIFY-RUNS/_HOW-RUN.txt">test notes</A>.
<DIV class=notebox>
<P class=noteboxheader>
<A name="speedmatters">Speed Matters</A>
<P>
Though most phones would lose a straight-up footrace
with a PC, on Android, Mergeall itself is relatively fast because it compares just
file timestamps and sizes by design. The more exhaustive <code>diffall.py</code>
utility, though, can run <I>much</I> longer for large content collections—which is
one reason for using incremental sync tools like Mergeall (though even-slower
full copies make it a slam dunk).
<P>
As an example, on-phone
comparisons of a 150G content collection having 100K files and 10K folders on
microSD and USB flashdrive work well, but take 2 <I>hours</I> in
<code>diffall.py</code> versus 2 <I>minutes</I> or less in Mergeall.
For long-running tasks like the former, be sure to acquire the Termux partial
<A HREF="https://developer.android.com/training/scheduling/wakelock">wake lock</A> in
Android's
<A HREF="_readme-items/PYDROID3-TKINTER/x-notific-keyboard-termux.jpg">notifications</A>
to keep your program running even after your screen
<A HREF="https://developer.android.com/reference/android/os/PowerManager.html#PARTIAL_WAKE_LOCK">blanks</A>.
Remember to release it when your task finishes to avoid battery drain.
<P>
And for <I>true</I> sports fans: the Mergeall comparison for the same 150G content
folder on the same drives clocks in at a quicker 24 and 54 seconds on Mac OS El Capitan
and Windows 7 machines, respectively—though the variables are many, and
2-minute on-phone merges still beat microSD-ectomies.
<P>
<SPAN class=update>Update</SPAN>:
results for <B>internal storage</B> are now in. On the same phone,
and using the same USB flashdrive and 150G content collection, Mergeall and
<code>diffall.py</code> comparisons to internal storage run in just 19..43 seconds
and 29 minutes, respectively. This is <I>3 to 4 times faster</I>
than microSD's results of 2 minutes and 2 hours on these same tasks.
Surprisingly, the <code>cpall.py</code> script copies the full 150G from USB to
internal storage
in just 21 minutes—slightly faster than a <code>diffall.py</code> comparison,
and perhaps indicative of a USB port and/or drive that's a bigger bottleneck than
internal-storage write speed. Either way, internal storage takes the gold in the
on-phone content sprint.
<P class=noteboxlast>
<SPAN class=update>Update</SPAN>:
in fall 2020, a Galaxy Note20 Ultra ran a full byte-for-byte <code>diffall.py</code>
comparison between a USB drive and internal storage in just 15 minutes—<B>twice
as fast</B> as the early 2019 Note 9 figure above, despite the fact that the content
set had grown to a substantially larger 194G. It's unknown if this reflects faster
drives, USB ports, CPUs, or combinations of these; but in computing, <I>waiting</I>
is often a completely valid work-around.
</P>
</DIV>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc41b><A name="fixfattimes">The FAT Times Fixer Tool</A></A></H3>
<P>
Additional helper scripts in this package run Mergeall's
<code>fix-fat-dst-modtimes-data.py</code>
to adjust file modification times on FAT32 drives so they are in sync
with non-FAT drives after daylight-savings-time (DST) changes.
Because FAT32 is recommended for broadest interoperability on Android
<A HREF="#toc22">today</A> (until Samsung <A HREF="#exfatepilog">Android 10</A>),
you may need to run this utility twice a year,
especially if you propagate changes to or from a PC or your phone's
internal storage.
<P>
Without getting into all the details here, on DST rollovers, FAT32's local
<A HREF="https://docs.microsoft.com/en-us/windows/desktop/SysInfo/file-times">times</A>
may all be one hour off from the UTC-based
<A HREF="https://en.wikipedia.org/wiki/Coordinated_Universal_Time">times</A>
used on non-FAT drives.
To diagnose this state, run Mergeall in report-only mode (its reams of messages
will show every file differing by modtime), and inspect the times of the same
file on both drives (they'll differ by one hour).
Less than ideal, perhaps, but cross-platform content
management is never dull.
<P>
Luckily, this state is also easily remedied. To make FAT32 times match other filesystems,
simply run the helper script as follows, with <code>data</code> for on-phone
storage and <code>plug</code> for USB drive; an <code>-add</code> or <code>-sub</code>
followed by a number-of-hours value; and an optional explicit logfile name:
<PRE>
$ gowork
$ bash mergeall-android-scripts/fix-fat-dst-modtimes-data.sh -add 1 logfile.txt
$ vi Admin-Mergeall/logfile.txt
</PRE>
<P>
In general, you'll want to run this script after DST rollovers on any FAT32 drives
whose times must compare correctly to non-FAT32 drives. Although the possible
combinations of filesystems and devices are many, most use cases fall into one
of the following categories:
<UL>
<LI>
If your on-phone content is in <I>internal storage</I>, run the fixer script on your FAT32 USB drives only.
On devices tested, internal storage uses UTC times just like PC internal drives, so updating an
intermediate FAT32 USB drive will bring it in sync with both your PC and your phone.
<LI>
If your on-phone content is on a <I>removable SD card</I> formatted as FAT32, run the fixer script on <I>both</I>
your FAT32 USB drives (to match your PC's UTC times), and your removable SD card (to match your intermediate
drive's fixed times).
<LI>
If you use Mergeall for <I>incremental backups only</I> and never propagate content to or from a
PC, a FAT32 USB drive must still be adjusted to match UTC-based internal storage or an exFAT-formatted
removable SD card, but need not be adjusted to match a FAT32 SD card.
<LI>
If you format your removable drives as <I>UTC-based exFAT</I> (despite the earlier <A HREF="#toc22">advice</A>),
they need not be adjusted to match UTC-based internal storage or exFAT-formatted SD cards, but an
adjustment is still required to make an exFAT USB drive match a FAT32 SD card.
</UL>
<P>
Adjust as needed for your usage, and see Mergeall's
<A HREF="https://learning-python.com/mergeall-products/unzipped/UserGuide.html#dst">user guide</A>
and the
<A HREF="https://learning-python.com/mergeall-products/unzipped/fix-fat-dst-modtimes.py">fixer script</A>
itself for more details.
You can also run the bare fixer script directly, either on your <I>PC</I> (to adjust times on an
attached USB drive or a removed SD card), or on your <I>phone</I> (to adjust times on an attached USB
drive or a resident SD card); its helper
<A HREF="fix-fat-dst-modtimes-data.sh">scripts</A> like the one shown in the example above
simply make on-phone runs easier. For instance, a direct run may look like this (try
<code>Mergeall-source</code> for <I>sourcepath</I> in your phone's <code>$work</code> folder,
and use <code>\</code> on Windows):
<PRE>
$ python <I>sourcepath</I>/fix-fat-dst-modtimes.py <I>folderpath</I> -sub 1 > <I>logfile</I>
</PRE>
<P>
On caution here: you should generally <I><B>restart your phone</B></I> before syncing content to
it after a DST rollover. Otherwise, Android may return file times saved earlier in a
cache that are now invalid, and will not compare correctly to other files' times.
This is arguably another Android bug; without a restart, its net result may be spurious
differences or equivalences, either one of which can derail Mergeall syncs.
<P>
For example, after adjusting times on a FAT32 USB drive up by an hour at a recent DST rollover
(March 2019 in the US), Android
initially and erroneously reported all the drive's times as <I>differing</I> from those on UTC-based
internal storage; a phone restart sufficed to make the drive's times agree with internal storage as
expected. Restarts may or may not be required for on-phone content on <I>removable</I> drives:
at the same DST rollover, FAT32 USB agreed with FAT32 SD and exFAT USB agreed with exFAT SD—without
restarts. To be safe, though, restart to prevent Android's error-prone caching from causing problems.
</P>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc42><A name=othertools>Other Mergeall Tools and Techniques</A></A></H3>
<P>
Mergeall's source-code package also includes the <code>cpall.py</code> folder-copy
script, and its <code>test</code> subfolder includes the <I>ziptools</I> package which
comes with zipfile create/extract scripts. If you elect to host your content on your phone's
internal storage instead of a removable card, for example, you might use either of
these to copy your content folder in full from a USB drive initially, or export
it to USB later.
<P>
Though not wrapped by helper scripts here, both of these tools can be run in your
<code>$work</code> folder with normal Unix command lines in Termux, using either
your bash-profile settings or custom paths. For instance, both of the following
command pairs import and export content copies on your phone ("py" is assumed to
be aliased to "python" in your bash profile file here, and routing the logfile is
up to you in this mode):
<!-- jan22: escape / with / for android opera bug -->
<PRE>
$ py Mergeall-source/cpall.py $plug $data -skipcruft -vv -u > Admin-Mergeall/importlog.txt &
$ py Mergeall-source/cpall.py $data $plug -skipcruft -vv -u > Admin-Mergeall/exportlog.txt &
$ py Mergeall-source/test/ziptools/zip-extract.py $plug/../import.zip $data
$ py Mergeall-source/test/ziptools/zip-create.py $plug/../export.zip $data/* -skipcruft
</PRE>
<P>
As usual, tab completion helps with input here, and bash scripts, variables, or aliases
can shorten such commands.
For more on using <code>cpall.py</code>, see the
<A HREF="https://learning-python.com/mergeall-products/unzipped/cpall.py">script</A>;
for more on ziptools, see its <A HREF="https://learning-python.com/ziptools.html">homepage</A>.
Termux itself comes with a Unix <code>unzip</code> and a
<code>pkg install zip</code> adds a Unix <code>zip</code>; both generally work
well, but are naturally subject to the Unix tools' constraints (e.g., file-size
limits may apply).
<P>
If you prefer a more interactive experience, Mergeall's
<A HREF="https://learning-python.com/mergeall-products/unzipped/UserGuide.html#console">console mode</A>
works on Android too. This mode asks you for input parameters, instead of expecting them
in the command line. It doesn't have tab completion for paths, though, so it's a bit
more typing or pasting, and may work better with a Bluetooth
<A HREF="https://wiki.termux.com/wiki/Hardware_Keyboard">keyboard</A>.
See the screenshots of this mode's
<A HREF="_readme-items/launch-mergeall-Console.png">setup</A> and
<A HREF="_readme-items/launch-mergeall-Console-results.png">results</A>,
and start it in Termux with a command like this:
<PRE>
$ py Mergeall-source/launch-mergeall-Console.py
</PRE>
<P>
Finally, you can also skip the helper scripts and bash profile altogether and run
Mergeall's scripts directly, as described in its
<A HREF="https://learning-python.com/mergeall-products/unzipped/UserGuide.html#cmdline">user guide</A>.
The following Unix command line, for instance, kicks off a Mergeall USB-to-SD updates
run on Android manually:
<PRE>
$ python Mergeall-source/mergeall.py \
/storage/7284-2735/Android/data/com.termux/MY-STUFF \
/storage/25C9-1405/Android/data/com.termux/MY-STUFF \
-auto -backup -quiet -skipcruft \
> ../Admin-Mergeall/mergeall-$(date +%Y%m%d-%H%M%S).txt &
</PRE>
<P>
And if you can imagine typing this all on a smartphone's touchscreen,
you'll probably appreciate why the helper scripts may be a better idea.
</P>
<!-- ============================================================================ -->
<H2><A name=toc5>Random Tips</A></H2>
<P>
This section collects assorted usage pointers and reminders.
Most are intended for users newer to Termux, Unix, or both.
</P>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc51>Termux Tips</A></H3>
<P>
<UL>
<LI>Swipe from the left and longpress Keyboard to display the control-key bar
<LI>Use New Session to work in multiple terminals (e.g., edit in one, run in another)
<LI>To edit text files on your phone, use <code>nano</code>, <code>vi</code>, or <code>emacs</code>
in Termux, or text-editor
<A HREF="https://play.google.com/store/apps/details?id=com.rhmsoft.edit">apps</A>
<LI><A HREF="_readme-items/nano-on-termux.png"><code>nano</code></A> is user friendly:
run <code>pkg install nano</code> to get, and tap <code>Alt+x</code> for more space
<LI>Longpress cut-and-paste works everywhere (e.g., to copy text to editors or commands)
<LI>The Termux Linux system is full-featured but "prefixed": its files live in <code>~/files</code>
(not in <code>/</code>)
<LI>Termux command lines and filenames are case sensitive: use uppers and lowers well
<LI>Termux's default <code>python</code> install is Python 3.7 today; install <code>python2</code> for 2.X code
(<A HREF="https://wiki.termux.com/wiki/Python">info</A>)
<LI><code>pkg install <I>X</I></code> installs Termux tools; <code>pip install <I>X</I></code> installs Python tools in Termux
<LI>To install Pillow: <code>pip install Pillow</code>, after running <A HREF="https://pillow.readthedocs.io/en/stable/installation.html#building-on-android">this command</A>
(omit its "-y" to approve changes)
<LI>For more on using Termux, see its usage pages
<A HREF="https://wiki.termux.com/wiki/User_Interface">here</A>,
<A HREF="https://wiki.termux.com/wiki/Touch_Keyboard">here</A>, and
<A HREF="https://wiki.termux.com/wiki/Hardware_Keyboard">here</A>
</UL>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc52>Other Tips</A></H3>
<UL>
<LI>Some, but not all, Android file explorer apps give removable-drive IDs;
<A HREF="https://play.google.com/store/apps/details?id=com.ghisler.android.TotalCommander">this</A> does
<!--
alas, blacklisted by google for ethics violations:
<A HREF="https://play.google.com/store/apps/details?id=com.estrongs.android.pop.pro">
-->
<LI>Home-screen shortcuts give quick access to content and working folders; see file explorers
<LI>Command-line tab completion and arrow recall are <I>essential</I> on phones; use them
<LI>Internal storage was 5X and 10X faster on tested writing tasks than a microSD card; use it
<LI>Internal storage was also 3X to 4X faster on tasks related to Mergeall
<A HREF="#speedmatters">specifically</A>; use it
<LI>Network drives with usable paths might provide other Mergeall options (not explored here)
<LI>Syncs between internal storage and removable SD cards work too (but use cases are fewer)
<LI>A Bluetooth keyboard and mouse/touchpad makes smartphone command-lines more
<A HREF="https://wiki.termux.com/wiki/Hardware_Keyboard">humane</A>
<LI>A Bluetooth mouse is nearly required for moves, resizes, and some
controls in tkinter <A HREF="#toc8">GUIs</A>
<LI>For thoughts on whether you <I>should</I> host content on mobile devices, see
<A HREF="https://learning-python.com/post-release-updates.html#mergeallandroid">this earlier post</A>
</UL>
</P>
<!-- ============================================================================ -->
<H2><A name=toc6>The Mergeall Termux Wrap-Up</A></H2>
<P>
In closing, if this Mergeall approach seems like a lot of work, it's because it is,
but it's also a price inherent in maintaining content on Android smartphones
with Termux command lines today. In exchange, this scheme provides a fully
functional incremental-backup and changes-propagation tool that you can run
directly on your Android device to synchronize with external drives—no
device "rooting" or card juggling required.
<P>
That being said, some users of both Mergeall and Android may prefer to remove
cards instead of using command lines, and this may be the only option for others.
Still, <I>any</I> scheme that can free your content from the constraints of
both closed devices and cloud storage is worth a look. Not only are storage
devices much quicker than clouds, they can automatically neutralize the
proprietary and even exploitive agendas that seem to underlie much of today's
smartphone experience.
<P>
However you choose to proceed, taking control of your digital property
is worth the effort.
</P>
<!-- ============================================================================ -->
<H2><A name=toc7>Appendix A: A Brief Primer on Android Pathnames</A></H2>
<P>
This appendix introduces and summarizes the folders used in this guide.
It's probably more useful to readers who haven't used command lines much
in the past, but also serves as a glossary of sorts. This appendix
was also updated in 2021 for some findings in Android 11 covered in full
<A HREF="https://learning-python.com/mergeall-android11-updates.html">off page</A>;
this spoils part of Appendix C's surprise, but makes this primer more useful.
<P>
Because the locations at which you'll store your content on Android
are central to the content-management approaches presented in this doc,
it's important to get a handle on these up front.
Android identifies a storage location just like every other platform, with
a <I>pathname</I>—a string of folder names that reflects folder nesting.
Some of Android's naming conventions are unique, though, and some pathnames
identify folders with specific roles. Here's a quick rundown on those that
play a part in the Mergeall story:
<!-- ---------------------------------------------------------------------------- -->
<DL>
<DT>App-private folder
<DD>
The Termux app used by this guide's approach installs and runs
its programs in its Android app-private folder. This folder is fully
private to the Termux app: it cannot be accessed by any other apps on
non-rooted phones, including file explorers, and is automatically deleted
when Termux is uninstalled. This makes it less than ideal for
general-purpose storage. The Termux app-private folder is
located at this pathname on your phone:
<PRE>
/data/data/com.termux
</PRE>
<P>
This folder is really located in internal storage, but is too hidden
to be accessed as such. Being Linux-based, Android uses the <code>/</code>
character to separate folders in a pathname like this; the one all the way to
the left means the device root (the top of the folder-nesting tree),
and the left-to-right order of <code>/</code>-separated names gives their nesting.
The <code>com.termux</code> part at the end is how Termux is known to Android;
other apps have similar dotted-path IDs—and similar app-private folders.
<P class=noteboxlast>
<SPAN class=update>Update</SPAN>:
as noted earlier,
Termux eventually added support for the Storage Access Framework (SAF),
which allows specially equipped file-explorer apps to access the Termux
app-private home folder from outside the Termux app. This isn't normal
access and subverts Android permissions, but it does allow broader use
of this otherwise private folder. For more details, see the
<A HREF="https://learning-python.com/android-deltas-scripts/_README.html#termux-app-private-SAF">later coverage</A> in <I>Android Deltas Sync</I>.
<!-- ---------------------------------------------------------------------------- -->
<DT>User-home folder
<DD>
Your Termux "home" folder lives in the Termux app-private folder too, at
the following pathname:
<PRE>
/data/data/com.termux/files/home
</PRE>
<P>
This is where you normally "are" when you first open Termux (before
you run a <code>cd</code> to go to a different folder).
It also goes by the names <code>~</code> and <code>$HOME</code>,
which are just shell syntax that automatically equate to this folder's
full pathname above.
Because you can't see this folder outside the Termux app, its scope and
utility are limited (sans tools covered in the update above).
Because of the way Termux works, though, this is
where you'll need to install and edit the bash profile file covered in this
<A HREF="#toc37">guide</A>.
<!-- ---------------------------------------------------------------------------- -->
<DT>Internal-storage folder
<DD>
Android allows all apps and their users to access and share available space
on the phone's internal storage freely (subject to permission requirements
for apps that we'll skip here for space). In fact, this space is often
called just <I>shared storage</I> in recognition of its cross-app role.
Confusingly, Android docs also sometimes call this storage "external" to
distinguish it from storage associated with a specific app, but this doc
prefers to use that term for physically removable drives.
<P>
Thanks to Android history—and the obfuscation magic of Linux links—there
are at least two pathnames that can be used to refer to internal shared storage on most
devices (but use the shorter unless you happen to enjoy pointless tapping):
<PRE>
/sdcard
/storage/emulated/0
</PRE>
Some devices provide alternate paths to this space that are less common.
And yes, <code>/sdcard</code> really means internal storage, not your
removable SD card, for legacy—and even tortuous—reasons we'll
omit here. Because internal storage is accessible to every app, and is
never automatically removed (short of a factory reset), it's
ideal for content storage if your phone has enough space. It's
also generally more convenient than your home folder for working in
Termux, because you're not limited to Termux tools.
<P>
If you opt to store both your
content and working folders in internal shared storage for best utility and
permanence, their pathnames might look like the following (though <code>/sdcard</code>
may have to be something else on some phones, and your stuff's <code>MY-STUFF</code>
may naturally vary):
<PRE>
/sdcard/MY-STUFF
/sdcard/work
</PRE>
<P>
<SPAN class=update>Update</SPAN>:
though uncertain at this writing, <B>Android 11</B> may impose
the same update constraints on internal storage that KitKat imposed
on removable media (described <A HREF="#toc25">earlier</A>), making it less flexible
than implied here. See <A HREF="#thefutureisclosed">Appendix C</A> for
more details.
<P>
<SPAN class=update>Update</SPAN>:
per the most recent
<A HREF="https://learning-python.com/mergeall-android11-updates.html#android11strikesback">news</A>,
most of internal shared storage will still be freely accessible to at least some
apps in <B>Android 11</B>, but will also run radically slower than internal app-private and
app-specific
<A HREF="https://learning-python.com/mergeall-android11-updates.html#asb5">storage</A>,
and app-specific folders within internal storage will be locked down to the owning
<A HREF="https://learning-python.com/mergeall-android11-updates.html#asb6">app</A>
(more on these folders ahead).
<!-- ---------------------------------------------------------------------------- -->
<DT>Removable-drive folders
<DD>
Removable media—including both SD cards and USB drives—show up under
the <code>/storage</code> root on Android when they are attached, with a
folder name derived from the drive's unique ID and composed of hex digits.
For example, here's one for a USB drive with an ID of <code>25C9-1405</code>:
<PRE>
/storage/25C9-1405
</PRE>
Your drives' IDs will differ (they're supposed to be unique, after
all), and you can find them in some Android file explorer apps after drives
have been attached (and the system has a chance to mount it in the
folder-nesting tree); see <A HREF="#toc52">earlier</A> for app pointers.
A physical drive's Android ID will change if it's reformatted,
but should generally remain constant otherwise. Either way, the drive's
path allows programs to perform device-agnostic processing of content over USB.
<P>
At least that's the removable-drive story on Samsung Android devices, and
it's prone to vary across vendors (it depends on how drives are mounted in
the underlying Linux system and vendor extensions). USB drives are also generally
available at their ID-based folders in <code>/mnt/media_rw</code> too, for instance,
but using that path may require special permissions or phone rooting that are
beyond this appendix's scope.
<P>
<SPAN class=update>Update</SPAN>:
per the most recent
<A HREF="https://learning-python.com/mergeall-android11-updates.html#asb4">news</A>,
<B>Android 11</B> imposes
new constraints on USB-drive access, making it less usable than
implied here.
Though prone to change,
general USB paths are revoked, requiring program
rewrites to use proprietary Java-based APIs instead of traditional POSIX techniques.
See the <A HREF="https://learning-python.com/mergeall-android11-updates.html#asb4">doc</A>
for possible work-arounds.
<!-- ---------------------------------------------------------------------------- -->
<DT>Removable-drive app-specific folders
<DD>
Like internal storage, removable drives are accessible to all apps (subject to
permission rules we'll skip here too). As covered in this guide,
though, Termux does not support <I>updates</I> to a removable drive except in the
drive's Termux app-specific folder. This folder isn't app private,
but it follows the same pathname format: if you store your content folder on
a removable drive, its pathname would look like the following (but use your
own drive ID and folder name,
instead of <code>25C9-1405</code> and <code>MY-STUFF</code>):
<PRE>
/storage/25C9-1405/Android/data/com.termux/MY-STUFF
</PRE>
<P>
Every app has one of these app-specific folders if it needs one; they are meant
for app-related content that might be too large or inconvenient to store
in the <code>/data</code> app-private folder.
As discussed in this guide, these app-specific folders are key to using
content on removable drives in Mergeall—whether from Termux command
lines, or the GUI of the next appendix. Storing content here enables Mergeall
updates on removable drives, though this folder's limited scope rules it out
for cross-app updates.
<P>
App-specific folders like these are also automatically
and implicitly deleted when the owning app is uninstalled. This is meant
to clean up an app's resources, but makes these folders subpar for storing
data. Because of that, you'll generally locate
your working folder elsewhere. Your Mergeall content would ideally be elsewhere
too, but updatable copies on limited-space phones may have no other option
than removable storage, and this requires app-specific folders today.
<!-- ---------------------------------------------------------------------------- -->
<!-- jan-2021 -->
<DT>Internal-storage app-specific folders
<DD>
For completeness, internal storage hosts app-specific folders too, which
follow the same pathname pattern as removable drives, and might look like
the following if your content is nested in Termux-specific storage:
<PRE>
/sdcard/Android/data/com.termux/MY-STUFF
</PRE>
<P>
This guide downplays this folder, because it's less functional: just like
their removable cousins, app-specific folders in internal storage are
automatically deleted when the owning app is uninstalled. Internal
app-specific folders are also more accessible than app-private space
(e.g., file explorers work), but not any more so than the rest of internal
storage. Hence, you're normally better off using less-transient locations
in internal storage for your content; anywhere else in <code>/sdcard</code>
is both safer and just as usable.
<P>
When this guide was written, app-specific folders in internal storage were
largely lumped together with the rest of internal space, because they
did not restrict updates to the owning app—unlike their removable kindred.
If this changes (and the following updates suggest it may), it will make
app-specific storage folders even less desirable than internal shared storage
at large.
<P>
<SPAN class=update>Update</SPAN>:
though uncertain at this writing, <B>Android 11</B> may impose
new rules on internal app-specific storage, making it less usable than
implied here. See <A HREF="#thefutureisclosed">Appendix C</A> for
more details.
<P>
<SPAN class=update>Update</SPAN>:
per later news, <B>Android 11</B> narrows the
<A HREF="https://learning-python.com/mergeall-android11-updates.html#fileexplorersupdate">accessibility</A>
of internal app-specific folders so much that they are generally unusable for content
managed by more than one app. Sans special new app
<A HREF="https://learning-python.com/mergeall-android11-updates.html#allfilesaccess">permissions</A>,
files to be processed in <I>both</I> Termux and Pydroid 3 must be kept in shared storage
(i.e., <code>/sdcard/<I>other</I></code>) instead.
<P>
<SPAN class=update>Update</SPAN>:
Android file-explorer apps ultimately found a loophole which allowed them to provide access to
app-specific folders in <B>Androids 11</B> and <B>12</B>. <B>Android 13</B> aims to
close this loophole, but the outcome of this is unclear at this writing. See the
<A HREF="https://learning-python.com/android-deltas-scripts/_README.html#android13-app-specific">later coverage</A>
in <I>Android Deltas Sync</I>.
<!-- ---------------------------------------------------------------------------- -->
<DT>Downloaded-content folder
<DD>
Finally, something simple: documents and packages you download in your web
browser normally wind up at the following location in internal storage
(or a similarly descriptive locale on other phones):
<PRE>
/sdcard/Download
</PRE>
<P>
This is where Mergeall's source-code and android-scripts zipfiles show up
if you fetch them from their web pages named in this <A HREF="#toc35">doc</A>.
Because internal storage is completely general and accessible, you can store
and use their unzipped content in this folder if you wish
(but watch for possible mods to this too in an Android 11
<A HREF="#thefutureisclosed">future</A>).
</DL>
<!-- ---------------------------------------------------------------------------- -->
<H4>A Storage-Paths Cheat Sheet</H4>
<P>
As a concise summary of the rules we've just met,
Android groups internal and removable storage into
accessibility and permanence categories as follows—all
subject to the Android device and version variability which
defies authoritative docs:
<style>
.hack {
margin-top: 7px;
margin-bottom: 7px;
}
</style>
<UL>
<LI>Internal <I>app-private</I> storage
<UL>
<LI class=hack>Lives at <code>/data/data/<I>appname</I></code>
<LI class=hack>Can be accessed normally only by the app
<LI class=hack>Is removed when the app is uninstalled
</UL>
<LI>Internal <I>app-specific</I> storage
<UL>
<LI class=hack>Lives at <code>/sdcard/Android/data/<I>appname</I></code>
<LI class=hack>Has access that varies by Android version
<LI class=hack>Is removed when the app is uninstalled
</UL>
<LI>Internal <I>shared</I> storage
<UL>
<LI class=hack>Lives at <code>/sdcard/<I>other</I></code>
<LI class=hack>Is generally accessible to all apps
<LI class=hack>Is never removed on app uninstalls
</UL>
<LI>Removable <I>app-specific</I> storage
<UL>
<LI class=hack>Lives at <code>/storage/xxxx-xxxx/Android/data/<I>appname</I></code>
<LI class=hack>Has access that varies by Android version
<LI class=hack>Is removed when the app is uninstalled
</UL>
<LI>Removable <I>shared</I> storage
<UL>
<LI class=hack>Lives at <code>/storage/xxxx-xxxx/<I>other</I></code>
<LI class=hack>Has access that varies by Android version
<LI class=hack>Is never removed with app installs
</UL>
</UL>
</P>
<!-- redundant...
<P>
This appendix wishes these rules could be tighter, but Android seems
a perpetually moving target.
-->
<!-- ---------------------------------------------------------------------------- -->
<P>
Keep in mind that all of these conventions pertain to Android only.
If you use the same removable drive across different platforms, the names
of its folders will differ on each. The following, for example, is the
pathname for the same folder on Android, Mac OS, Linux, and Windows;
all reference the same folder, but you'll need to extrapolate from
the Android flavor shown in this guide when using a drive elsewhere:
<PRE style="line-height: 125%;">
<I>Android</I>: /storage/25C9-1405/Android/data/com.termux/MY-STUFF
<I>Mac OS</I>: /Volumes/EXT256-1/Android/data/com.termux/MY-STUFF
<I>Linux</I>: /media/<I>username</I>/EXT256-1/Android/data/com.termux/MY-STUFF
<I>Windows</I>: D:\Android\data\com.termux\MY-STUFF
</PRE>
<P>
And again, also keep in mind that the pathname forms above and used in this
guide may vary on some Android devices, and have varied over Android's history.
For instance, <code>/storage/extSdCard</code> and <code>/storage/sdcard0</code>
worked in the past too. The paths used here mostly reflect Samsung Galaxy
devices using Androids 7 (Nougat) through 10, so you'll have to translate
if those on your phone differ. Alas, Mergeall's budget is limited, and
stability is not among Android's core virtues.
<P>
For more details on Android pathnames, see books, the web, or other Android
resources.
</P>
<!-- ============================================================================ -->
<H2><A name=toc8>Appendix B: The Pydroid 3 tkinter GUI Alternative</A></H2>
<!-- +++++++++++++++++++++++++++PPUS announce+++++++++++++++++++++++++++++++++ -->
<!-- copied from later and separate andoid tkinter page -->
<DIV class=ppusannounce>
<P>
<I><B>User alert</B></I>: as of its August 2023 version 6.4, the
<A HREF="https://play.google.com/store/apps/details?id=ru.iiec.pydroid3">Pydroid 3</A>
app presented here for running tkinter GUIs is much less usable for programs
that share content with other apps. You can still use it to experiment with
Python and tkinter without a PC, but its loss of shared-storage access means
that you shouldn't expect to use it for most realistic work, and should
explore standalone-app builders as an alternative for Python on
<A HREF="https://quixotely.com/PC-Phone%20USB%20Sync/">Android</A>.
For the full story, please visit
<A HREF="https://learning-python.com/pydroid3-loses-storage-access.html">this page</A>.
</DIV>
<!-- +++++++++++++++++++++++++++END PPUS announce+++++++++++++++++++++++++++++ -->
<P>
The main part of this guide recommends and covers running
<A HREF="https://learning-python.com/mergeall.html">Mergeall</A> by using command lines
in the <A HREF="https://termux.com/">Termux</A> app, because this approach is both
straightforward and reliable, and is completely advertising and payment free.
Termux provides a full-featured and general-purpose Linux environment, which
can run Mergeall's command-line mode with minimal setup tasks and robust performance.
This is not, however, the only Python-on-Android game in town.
<P>
Most notably, the
<A HREF="https://play.google.com/store/apps/details?id=ru.iiec.pydroid3">Pydroid 3</A>
app has recently acquired some rudimentary though impressive support
for using the Tk GUI library—and Python 3.X's <I>tkinter</I>
interface to it—in Python programs. This makes it possible to use
Mergeall's GUI on Android devices today, along with a host of other existing
tkinter software. Less positively, this app's tkinter support also comes with
substantial downsides, including both behavioral glitches and unfortunate
advertising, that make this approach difficult to recommend in this guide.
<P>
That being said, the prospect of running PC GUIs on a smartphone is
arguably seductive, and the possibilities that this app raises
for Mergeall users merit both fair review and your own judgement. You may or may
not prefer Pydroid 3 GUIs to Termux command lines in the end, but this
appendix provides enough coverage of the former to help you make a good
usage-mode choice.
<DIV class=notebox>
<P class=noteboxheader>
Version Notes
<P>
The findings in this appendix reflect <B>Pydroid 3</B> versions 2.22 and 3.0;
<B>Python</B> versions 3.6 and 3.7; an Android port of <B>Tk</B> version 8.6;
and <B>Android</B> versions 7 though 9 (Nougat, Oreo, and Pie). All four of
these systems play roles in issues noted along the way. While this how-to hopes
to be revised for fixes in these systems, change is constant and rapid in the
software world. If you're reading this in the future, be sure to check all
three for current status. With any luck, the glitches list here will be
completely invalid by the time you see it.
<P class=noteboxlast>
<SPAN class=update>Update</SPAN>:
as of fall 2020, Mergeall's GUI has also been verified to run on <B>Pydroid 3</B> version 4.0,
the <B>Python</B> 3.8 used by that app, and <B>Android</B> 10 (formerly called Q). The glitches
listed <A HREF="#toc85">ahead</A> were retested and updated for this context (if not marked
"(Fixed)" they're still present). Though
not germane to this appendix, Mergeall command lines also run on the latest Termux
and its Python 3.8.
All told, this means you can run Mergeall and its GUI on Androids 8 (Oreo), 9 (Pie),
and 10; Nougat is ruled out by its
timestamp <A HREF="#toc21">bug</A>.
Android 11 support is still murky, given its planned change to internal-storage
<A HREF="#toc9">permissions</A>; Mergeall will likely work on 11, but
content may have to be nested barring app work-arounds.
</P>
</DIV>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc81>Mergeall's GUI in Action</A></H3>
<P>
First and foremost, Mergeall's GUI <I>does work</I> in Pydroid 3 with only minor
changes, and without the command lines required for Termux use. Under this
alternative approach, your content syncs will be run in the Pydroid 3 app's IDE
(its edit+run GUI) that looks like
<A HREF="_readme-items/PYDROID3-TKINTER/pydroid3-IDE.jpg">this</A>,
and the Mergeall GUI you will use to configure and run syncs on
your phone will look as follows:
<DL>
<DT>Portrait orientation
<DD>
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-portrait-1.jpg">Start</A>,
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-portrait-2.jpg">browse</A>,
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-portrait-3.jpg">report</A>,
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-portrait-4.jpg">finish</A>,
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-portrait-5.jpg">update</A>,
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-portrait-6.jpg">finish</A>
<DT>Landscape orientation
<DD>
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-landscape-1.jpg">Start</A>,
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-landscape-2.jpg">browse</A>,
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-landscape-3.jpg">report</A>,
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-landscape-4.jpg">finish</A>,
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-landscape-5.jpg">update</A>,
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-landscape-6.jpg">finish</A>
<DT>Other viewing modes
<DD>
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-nonmax-landscape.jpg">Non-maximized</A>,
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-nonmax-portrait.jpg">Non-maximized</A>, and
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-nonmax-nonfull-portrait.jpg">Non-fullscreen</A>
</DL>
<P>
You can also view the screenshots above in slideshow mode
<A HREF="_readme-items/PYDROID3-TKINTER/_thumbspage/mergeall-landscape-1.jpg.html">here</A>.
Despite the tradeoffs we'll explore in this appendix, it's important to note
that running tkinter programs this way is an enormous achievement,
given the large disparities between PC and Android platforms.
<H4><A name="Pydroid 3 Viewing Modes">Pydroid 3 Viewing Modes</A></H4>
<P>
Now that you've seen Mergeall's GUI at work, we need to define some terms that impact
the way it renders in Pydroid 3.
The first two sets of screenshots above were captured with this app's tkinter
<I>fullscreen</I> and <I>maximized</I> viewing modes enabled, by setting the
first "Tkinter" switch off and the second on in the app's Settings ⇨ System
<A HREF="_readme-items/PYDROID3-TKINTER/pydroid3-viewing-modes.png">dialog</A>.
You can use neither, either, or both of these modes, but their utility can
vary per GUI and usage:
<UL>
<LI><I>Fullscreen</I> mode saves space by hiding the Android top-of-screen
status bar. But it may cause conflicts with Android's notifications pulldowns
whenever you tap near the top of the screen (especially when trying to move
windows with touch).
<LI><I>Maximized</I> mode uses space well by expanding the first-launched
window to fill the entire display without a window border, and automatically
resizes the window to fit the new aspect ratio on each rotation. But it doesn't
work for persistent popup windows (they're overlaid and hidden), and is dangerous
or unusable for programs that need to process or verify exits (the double-back
tap alternative to a window-border's "X" exit button does not run
program exit handlers).
</UL>
<P>
Despite their downsides, Mergeall seems to work best with both these modes enabled,
because it has no persistent popup windows and doesn't need to catch program exits, and
because screen space is a precious commodity on phones in general. Mergeall works
without these modes too—and looks like the last set of screenshots above—but
has less space, and may bear manual resizing which seems out of place on a phone.
<H4><A name="guiusage">General Usage Notes</A></H4>
<P>
As a more general observation, tkinter GUIs in Pydroid 3 can take a bit of getting
used to, and some are substantially easier to operate with a Galaxy Note-type stylus
or Bluetooth mouse. For instance:
<UL>
<LI>There is no <I>window manager</I>, apart from the IDE's canvas:
multiwindow programs and window resizes are supported, but there is no minimize,
and the IDE can run only one program at a time.
<LI>As shipped, <I>rightclicks</I> are drive-by swipes instead of press-and-holds or
mouse rightclicks. In programs that support it, either swipe with touch or stylus,
swipe with a mouse cursor while holding its left button, or use a
control-key + mouse-click combo with a Bluetooth keyboard.
<LI>Some <I>smaller controls</I> are difficult to operate with touch alone,
and may benefit much from a stylus, mouse, or two-finger spread/pinch <I>zooms</I> which
expand/shrink the view generally and three-finger <I>pans</I> which scroll zoomed views.
<LI>Window <I>resizes and moves</I> for multiwindow GUIs run in non-maximized
mode are frustrating to perform with touch, and nearly impossible without a stylus
or mouse (or the patience of a saint).
<LI>Some larger GUIs may warrant, if not require, <I>landscape</I> orientation for better
fit, <I>maximized</I> viewing mode to avoid display overflow and slides,
and <I>navigation bar</I> hiding and/or <I>fullscreen</I> mode for extra space.
<LI>Some larger GUIs may also benefit from lower Pydroid 3 <I>DPI settings</I> that
shrink the GUI in full, thereby allowing for more on-screen content. For more
details on this option and GUI-to-display fit in general see the related doc's
<A HREF="https://learning-python.com/using-tkinter-programs-on-android.html#moreonfit">sidebar</A>.
<LI>Though subjective, most tkinter GUIs reflect their PC heritage, and some of their
<I>cosmetics</I> may seem foreign to users accustomed to apps designed specifically for
smaller Android devices.
</UL>
<P>
On the other hand, tkinter GUIs are surprisingly usable on a phone with
practice—and might even be compelling.
<DIV class=notebox>
<P class=noteboxheader>
<A name="androwish">Is That AndroWish under the Hood?
<P>
Programmers may be interested to know that the Pydroid 3
app's tkinter support <I>might</I> be based on the work of
<A HREF="http://www.androwish.org">AndroWish</A>—an open-source
project that uses the
<A HREF="https://en.wikipedia.org/wiki/Simple_DirectMedia_Layer">SDL</A>
graphics library to render Tk GUIs on Android devices, employs a
<A HREF="https://en.wikipedia.org/wiki/Java_Native_Interface">JNI</A>
bridge between Android's Java code and Tk's C
code, and was itself based on an earlier project called SDLTk
(whose web whereabouts are unknown).
If so, this means that Pydroid 3's tkinter simply inherits both its utility
and flaws from AndroWish, though the Pydroid 3 commercial product also shares
responsibility for the software it bundles and sells.
<P>
Unfortunately, Pydroid 3's developer did not reply to <!-- multiple -->
requests for verification of AndroWish use.
To be sure, Pydroid 3's tkinter bears multiple signs of AndroWish-ness.
Among them:
its two-finger zooms and three-finger zoomed pans work in AndroWish too;
its unique rightclick bindings smack of AndroWish's
<code>sdltk touchtranslate</code> command;
its GUIs-cannot-spawn-GUIs constraint discussed ahead also exists in AndroWish;
its tendency to fire multiple events for one gesture parallels an AndroWish bug report;
its install tree includes SDL components used by AndroWish;
and its overall look-and-feel seems the same as AndroWish's.
Still, this isn't enough evidence for a positive identification.
<P>
What can be said is that Pydroid 3's tkinter is likely based
on work of others that is not necessarily tied to Pydroid 3, and might be
usable in other Python coding apps, in standalone apps of your own, and
even on other
<A HREF="https://www.androwish.org/index.html/wiki?name=undroidwish">platforms</A>.
While intriguing, such prospects are both beyond this guide's scope, and
largely moot. The more important point here is that Pydroid 3 today brings Tk
and tkinter on Android to the broad Python audience—something that the
Tcl-biased AndroWish project seems unlikely to have ever pursued. Hence, we'll
attribute Pydroid 3's tkinter to Pydroid 3 here because it's no less a part of this
app than any other component it embeds, and note that running tkinter GUIs
like Mergeall's on Android is a stellar achievement, regardless of its implementation.
<P class=noteboxlast>
<SPAN class=update>Update</SPAN>:
after a brief tour of build options and code, it appears that the
<B>SDLTk</B> library may be all that is required to enable tkinter
in Python scripts on Android (along with the SDL library which SDLTk uses,
and substantial Java-related anguish).
<I>AndroWish</I> by itself adds extra utility for Tcl programs, which may not
be as valuable to Python code. That said, the most usable SDLTk version
<A HREF="http://www.ch-werner.de/sdltk/">available today</A>
was also updated and posted by the developer of AndroWish, so it seems
reasonable to call the two a set. Given SDLTk's separate origin, though,
it also seems reasonable to call Pydroid 3 a sort of AndroWish for Python.
</P>
</DIV>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc82>Mergeall Pydroid 3 Requirements</A></H3>
<P>
The prior section paints a justifiably positive picture, but tkinter
GUIs do not come without both procedural requirements and noticeable drawbacks
in Pydroid 3. To bring Mergeall's GUI to life on your phone, you must first be
willing to accept some constraints unique to Pydroid 3, and others
common to Android in general. Specifically, you must:
<UL>
<LI>Install the Pydroid 3 app on your phone: see its Play Store <A HREF="https://play.google.com/store/apps/details?id=ru.iiec.pydroid3">page</A>
<LI>Fetch and unzip Mergeall's source-code package on your phone from <A HREF="https://learning-python.com/mergeall.html">here</A>
<LI>Replace two of Mergeall's source-code files with versions that work around Pydroid 3 issues
<LI>Use Mergeall by opening and running its GUI's source-code file in Pydroid 3's IDE on every sync
<LI>Accept a few remaining Pydroid 3 idiosyncrasies and flaws that cannot be addressed in code
<LI>Pay Pydroid 3's $9.99 freemium fee, or tolerate its rude pay-or-else fullscreen ads
<LI>Satisfy all the general requirements described earlier for the Termux approach
</UL>
<P>
Compared to the Termux approach, you do not need to install Python separately (it's part
of Pydroid 3); install helper scripts (you'll use the GUI instead); or run a permissions
setup command (Pydroid 3 requests permissions automatically).
<P>
However, you still need
Mergeall's source-code package (you'll run its GUI's script); must patch two of its code
files for Pydroid 3 alone (per the next section); and still have to work with Android
version, filesystem, and permission constraints (per their earlier <A HREF="#toc2">coverage</A>).
<P>
While some of the steps above are self-explanatory, and we don't have space for an exhaustive
dissection of the rest, the remainder of this appendix explores the most-salient points on
the list above.
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc83>Required Mergeall Code Changes</A></H3>
<P>
Among the requirements just listed, the third may merit explanation most.
To use Mergeall's GUI in Pydroid 3, you must first download
Mergeall's <I>source-code package</I> from
<A HREF="https://learning-python.com/mergeall.html">this page</A> to any
writeable folder on your phone, and unzip (i.e., extract) it there.
Internal storage's <code>/sdcard/Download</code> is the usual target for
downloads, and will suffice for both a quick look and later syncs.
There's more on Mergeall's source-code package
<A HREF="#toc35">here</A> and <A HREF="#toc23">here</A>
if you've jumped into this appendix randomly.
<P>
After installing the Mergeall source-code package, you'll need to <I>replace two
of its files</I>, because it's not possible to integrate Pydroid 3's changes
into the source-code package today for reasons explained <A HREF="#toc86">ahead</A>.
To do this, simply click the following links to fetch custom versions of two Mergeall
source-code files, and move or copy them into the top level of your unzipped Mergeall
source-code package folder on your phone.
That is, replace the unzipped originals with the versions below:
<UL>
<LI><A HREF="_readme-items/PYDROID3-TKINTER/CODE/launch-mergeall-GUI.pyw"><code>launch-mergeall-GUI.pyw</code></A>
<LI><A HREF="_readme-items/PYDROID3-TKINTER/CODE/mergeall_configs.py"><code>mergeall_configs.py</code></A>
</UL>
<P>
If you click these links online, you'll receive reply pages that looks like
<A HREF="_readme-items/PYDROID3-TKINTER/save-changed-files.png">this</A>;
download to your phone by clicking the "save" in these pages' "Raw text" lines.
If you're working offline, try a rightclick/longpress save,
or visit the folder these links reference.
<I>Tip</I>: some browsers may append a bogus ".txt" to the end of a saved ".pyw"
file's name; delete the ".txt" manually by renaming if needed.
<H4>What Was and Wasn't Changed</H4>
<P>
The changes applied in these files are minor, and apply to Mergeall's GUI only. Hence,
they won't break Mergeall's command-lines mode described earlier in this guide if it's
run from the same modified source-code folder. Conversely, these changes will break Mergeall
on PCs, so use the files above only on your phone's copy of the source-code package.
All changes made are marked with "# ANDROID" comments; search for this string in the
custom files if you want to see what differs on Android.
<P>
In short, these files' changes address some but not all limitations of tkinter GUIs in Pydroid 3.
For example, text labels were trimmed and control fonts shrunk for better fit;
some size settings were lowered for non-maximized mode; a workaround was installed
for a Pydroid 3 <code>sys</code> module glitch that prevented the GUI from spawning merges;
other workarounds were installed for Python <code>webbrowser</code> issues that precluded
document views; a button was decolorized to avert color loss; and later releases added
dialog-text breaks, folder-choice prefills, and read-only text state, to avoid message
truncation, chooser dialogs, and keyboard popups, respectively.
<P>
Despite all the changes applied for Android,
though, you must still avoid using some fonts in customizations because they trigger crashes
prior to Pydroid 3 3.0 and force blind choices later,
and should not rotate some devices while Mergeall is running because this can hang or kill
Pydroid 3 altogether (until its <A HREF="#rotationsglitch">version 4.0</A>).
<P>
We'll take a closer look at these and other glitches in Pydroid 3's tkinter GUIs in
a moment. First, though, let's run the GUI.
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc84>How to Run Syncs in Mergeall's GUI</A></H3>
<P>
Once you've made the prior section's code changes, start Mergeall's GUI by opening
its source-code package's file <code>launch-mergeall-GUI.pyw</code> in the
Pydroid 3 app's editor, and pressing the editor's big yellow run
<A HREF="_readme-items/PYDROID3-TKINTER/pydroid3-IDE.jpg">button</A>.
<P>
Mergeall's GUI should
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-landscape-1.jpg">appear</A>,
and work the same as it does on PCs and as described
<A HREF="https://learning-python.com/mergeall-products/unzipped/UserGuide.html#gui">here</A>—select FROM and TO folders,
choose a REPORT or UPDATE, pick a logfile folder and other run options,
and press the GUI's GO button to sync.
Mergeall messages scroll in the GUI,
and are saved to a logfile if you've requested one. As usual, you can open
Mergeall's source-code file in either Pydroid 3 itself (see its folder
<A HREF="_readme-items/PYDROID3-TKINTER/pydroid3-IDE.jpg">icon</A>),
or most Android file explorers (tap the file and open with Pydroid 3).
<H4>Configuring Pydroid 3</H4>
<P>
As noted earlier, Mergeall tends to work best in fullscreen and maximized modes.
If desired, select these modes before running the GUI with the Settings ⇨ System
<A HREF="_readme-items/PYDROID3-TKINTER/pydroid3-viewing-modes.png">dialog</A>,
and press your phone's back button twice when you wish to exit Mergeall in
maximized mode.
While you're in Pydroid 3's Settings, also consider using the Editor's monospace
font without line wrapping; Mergeall's code is otherwise difficult to view.
Viewing code is not required for the GUI, but you may wish to experiment with
configuration-file settings in
<A HREF="_readme-items/PYDROID3-TKINTER/CODE/mergeall_configs.py"><code>mergeall_configs.py</code></A>,
and the line between user and programmer is often gray at best.
<P>
<SPAN class=update>Update</SPAN>:
two related tips here. First,
unlike Termux's run-on-demand
<A HREF="#toc32">tool</A>, Pydroid 3's <B>extra-permissions request</B> popup
has historically been triggered by a first permission error; if Python
scripts generate permission errors in Pydroid 3, it may help to create a new file in
the app's IDE and save it to a folder outside Pydroid 3's own, to force the permissions
request. Also, there's a new sidebar in the companion
<A HREF="https://learning-python.com/using-tkinter-programs-on-android.html#moreonfit">tkinter doc</A>
with info on setting Pydroid 3's <B>DPI settings</B>; this can improve GUI-to-display fit.
<H4>Other Mergeall Tools</H4>
<P>
This GUI approach works, but only for Mergeall syncs: other Mergeall
tools like <code>diffall.py</code> comparisons and <code>cpall.py</code> bulk copies
have no GUI, and are still best run with command lines in the Termux shell as covered
<A HREF="#toc4">earlier</A>.
Pydroid 3 does have a Terminal option that is capable of running command lines too
(and can even run the main Mergeall script with a command typed
<A HREF="_readme-items/PYDROID3-TKINTER/pydroid3-mergeall-commandline.png">manually</A>),
but it's much more limited than Termux for command-line work.
And while you can open and run such scripts in Pydroid 3's IDE, they won't work because
the IDE has no way to specify the command-line arguments they require.
<P>
As a special case, running Mergeall's GUI to sync to an <I>empty folder</I> has the
same effect as <code>cpall.py</code> (and in fact uses the same code), but you won't
get as many status messages, and there's no GUI equivalent to <code>diffall.py</code>
today. Command-line scripts need command lines (or an IDE that supports
<A HREF="_readme-items/PYDROID3-TKINTER/others-pyedit.png">them</A>).
<DIV class=notebox>
<P class=noteboxheader>
<A name="pydroid3shortcuts">Run GUIs Faster with Shortcuts</A>
<P>
Opening and running a source-code file in Pydroid 3's IDE normally requires multiple
steps—you must either open the app and navigate to your file, or navigate to
your file in a file explorer and open with the app. This, plus clicking the IDE's
run button, makes for substantially more steps than either a command in Termux or a
standalone app (and is one reason this guide downplays the GUI option).
<P>
That said, some Android file-explorer apps
allow you to place a shortcut (a.k.a. link)
to a source-code file on your home screen, like
<A HREF="_readme-items/PYDROID3-TKINTER/program-file-shortcuts-1-es.jpg">this</A> and
<A HREF="_readme-items/PYDROID3-TKINTER/program-file-shortcuts-2-tc.png">this</A>.
Once placed, starting a tkinter GUI program like Mergeall in Pydroid 3 is just
<I>two taps</I>—click the shortcut, and click the IDE's run button. Pydroid 3
still has extra setup steps and glitches, but running a tkinter GUI can be nearly as
easy as running an app.
Usage notes: file explorers like
<A HREF="https://play.google.com/store/apps/details?id=com.ghisler.android.TotalCommander">this</A>
<!--
alas, blacklisted by google for ethics violations:
<A HREF="https://play.google.com/store/apps/details?id=com.estrongs.android.pop.pro">
-->
are among those that make links,
and be sure to set the default app or command for Python files (both <code>.py</code>
and <code>.pyw</code>) to be
Pydroid 3 in your file explorer before creating the shortcut.
<!-- this was funny once...
(unless you like surprises more than monkeys).
-->
<P class=noteboxlast>
<SPAN class=update>Update</SPAN>: if you use the latest 3.0 version of the
<B>Total Commander</B> file explorer noted throughout this
<A HREF="https://play.google.com/store/apps/details?id=com.ghisler.android.TotalCommander">page</A>,
be careful to uncheck the
"Parameters" box when making internal associations and home-screen shortcuts.
Else, the <code>content://</code> URIs sent to Pydroid 3 fail to open the target
Python source file. It's unclear if this is a Total Commander or Pydroid 3 issue,
but the <code>file://</code> URLs sent by the former's pre-3.0 versions still work
well on Android 10 in the latter's current 4.0 release. This doc formerly also
noted <I>ES File Explorer Pro</I>, but that app was removed from the Play store for
advertising
<A HREF="https://duckduckgo.com/?q=es+file+explorer+removed+from+play+store">violations</A>;
good thing Google sacks this sort of thing...
</P>
</DIV>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc85><A name="pydroidglitches">Glitches in Pydroid 3 tkinter GUIs</A></A></H3>
<P>
Now that you know how to run a GUI sync on Android, you should also be aware up
front that some of Pydroid 3's tkinter support may still not quite be ready for
prime time. Running tkinter GUIs on Android is undoubtedly a major accomplishment,
but the current implementation still has rough edges that Mergeall code changes
cannot entirely smooth. In the interest of full disclosure, this section covers
the most prominent technical glitches known to confront Pydroid 3 tkinter GUIs.
<P>
Three brief scope notes about the items described here: some don't impact Mergeall,
but do impact other programs; some are probably of more interest to programmers
than to end users, though the distinction for source-code programs is gray;
and some have been repaired as of this doc's latest update, and are marked
"(Fixed)" in this list. Also note: if JavaScript is enabled in your browser,
a "Show Index" button below toggles a succinct index list summarizing this
section's content:
<!-- ====================== -->
<!-- does this warrant on index list? there are 21 items, and they change... -->
<!-- opted to make it a toggled option, iff JS enabled, and button clicked/tapped -->
<!-- don't show if no JS: wouldn't work (enabled iff JS ahead) -->
<P>
<button id=glitchbutton
style="display: none;"
onclick="showGlitchIndex();">Show Index</button>
<!-- don't show initially (or ever, if no JS -->
<P>
<DIV id=glitchindex style="display: none; margin-bottom: 25px;">
<UL>
<LI><A HREF="#_gl1">GUIs may require redesign for fit</A>
<LI><A HREF="#_gl2">(Partly fixed) Font support is limited and crashy </A>
<LI><A HREF="#_gl3">(Fixed) Phone rotations may kill or hang GUIs</A>
<LI><A HREF="#_gl4">(Fixed) Phone rotations during activities misplace or hang GUIs</A>
<LI><A HREF="#_gl5">Python's webbrowser works marginally in Pydroid 3's 2.2 release</A>
<LI><A HREF="#_gl6">Python's webbrowser is fully broken in Pydroid 3's 3.0 release</A>
<LI><A HREF="#_gl7">Python's multiprocessing doesn't work on Android (but threads do)</A>
<LI><A HREF="#_gl8">(Fixed) Pydroid 3 breaks spawns by leaving sys.executable empty</A>
<LI><A HREF="#_gl9">GUIs cannot spawn GUIs</A>
<LI><A HREF="#_gl10">Pydroid 3 is picky about imports</A>
<LI><A HREF="#_gl11">Pydroid 3 crashes badly on larger source files</A>
<LI><A HREF="#_gl12">Double-back exits don't run exit handlers</A>
<LI><A HREF="#_gl13">Newlines may need help</A>
<LI><A HREF="#_gl14">Keypresses may need help</A>
<LI><A HREF="#_gl15">Keypresses may need silencing</A>
<LI><A HREF="#_gl16">Keypresses botch shifts in some programs</A>
<LI><A HREF="#_gl17">Copy/paste may need help</A>
<LI><A HREF="#_gl18">Buttons glitch #1: losing background color</A>
<LI><A HREF="#_gl19">Buttons glitch #2: getting stuck</A>
<LI><A HREF="#_gl20">Android issues cast a wide net</A>
<LI><A HREF="#_gl21">And so on</A>
</UL>
</DIV>
<script>
var glitchIndexDisplay = document.getElementById('glitchindex')
var glitchIndexButton = document.getElementById('glitchbutton')
function showGlitchIndex() {
//
// On glitched-display button: open/close index display at section top (toggle)
//
if(glitchIndexDisplay.style.display == "block") {
glitchIndexDisplay.style.display = "none"; // hide index
}
else {
glitchIndexDisplay.style.display = "block"; // show index
}
}
glitchIndexButton.style.display = "block"; // show button now/always iff JS (here)
</script>
<!-- ====================== -->
<DL>
<DT><A name="_gl1">GUIs may require redesign for fit</A>
<DD>
Some PC GUIs may be too large to fit well in the smartphone form factor,
and require changes to avoid widget truncation in maximized mode, or horizontal
slides in non-maximized mode. In Mergeall, this meant trimming text labels and
shrinking control fonts for better fit on smaller screens, and lowering some
preset size configurations for non-maximized mode (though they're irrelevant at
maximized size).
<P>
Getting the fit right for smaller phones can be difficult.
For example, because Pydroid 3's tkinter naively clips nearby widgets instead
of truncating the text of labels that don't fit, Mergeall's original and descriptive
PC-GUI
<A HREF="_readme-items/PYDROID3-TKINTER/x-glitches-labels-PC-full.png">labels</A>
were initially
<A HREF="_readme-items/PYDROID3-TKINTER/x-glitches-labels-5.5-orig.png">unusable</A> on
phones of all sizes, and had to be trimmed
<A HREF="_readme-items/PYDROID3-TKINTER/x-glitches-labels-5.5-try2.png">radically</A>
for use on screens as small as 5.5"
(and may still be too wide on even smaller displays without new user
<A HREF="https://learning-python.com/using-tkinter-programs-on-android.html#mergeall-labels">configs</A>).
<P>
More-sophisticated tkinter GUIs may naturally require more redesign work.
This is especially true for those that open multiple active windows at once—a
desktop-metaphor paradigm at odds with both Pydroid 3's rendering options and Android's
one-window-at-a-time app model. While new tkinter GUIs can be designed with
such models in mind, many existing GUIs' shrinkability will be firmly limited
by usability; Mergeall might someday run on your watch, but it probably shouldn't.
<P>
<span class=update>Update</span>: it's worth adding that Android GUIs can also
be impacted by global <B>user settings</B> for font size, screen zoom, and display resolution;
GUIs have no control over these, and might not accommodate pathological cases.
Pydroid 3 tools such as maximized mode and low DPI settings can often improve
rendering, but users share some responsibility on this front. For tips on better
fitting GUIs to displays, see the related doc's
<A HREF="https://learning-python.com/using-tkinter-programs-on-android.html#moreonfit">sidebar</A>.
<DT><A name="_gl2">(Partly fixed) Font support is limited and crashy</A>
<DD>
Font support in Pydroid 3's tkinter is weak enough to ensnare many PC GUIs,
including Mergeall's. In general, font-family <I>names</I> <code>courier</code>,
<code>times</code>, and <code>helvetica</code>
work (and <code>monaco</code> is <code>courier</code>, and <code>arial</code> is
<code>helvetica</code>), but most other font family names trigger an immediate
and silent hardcrash (including <code>system</code>). Moreover, font <I>sizes</I>
work as expected, but italic and bold font <I>styles</I> are simply ignored if used
for <code>courier</code> (though they work for other font families).
To test this for yourself, edit and run
<A HREF="_readme-items/PYDROID3-TKINTER/CODE/_font-tests-pydroid3.py">this file</A>
in Pydroid 3.
<P>
Mergeall avoids crashes with font presets tailored for Pydroid 3 to use only
supported fonts, but this constrains functionality, and is error prone in the
context of end-user settings. You can still tailor
its fonts in the configurations
<A HREF="_readme-items/PYDROID3-TKINTER/CODE/mergeall_configs.py">file</A>,
but customize with care (and see
<A HREF="_readme-items/PYDROID3-TKINTER/CODE/_android_test.py">this file</A>
for names to use with platform-specific settings like fonts).
<P>
<span class=update>Update</span>: Pydroid 3's 3.0 release in early April 2019
<B>fixed its font crash</B> by coercing any unknown font-family name to <code>helvetica</code>,
but its other font limitations described above are still present. That is,
unknown fonts no longer crash GUIs silently, but <code>helvetica</code> is
substituted blindly, and no additional family names (or <code>courier</code>
styles) are now supported. It's a fix of sorts, but the app's tkinter clearly
requires more font work.
<DT><A name="_gl3"><A name="rotationsglitch">(Fixed) Phone rotations may kill or hang GUIs</A></A>
<DD>
Even more glaringly, on some devices, rotating the screen between portrait and landscape
orientation while any tkinter program is running either kills the GUI immediately and
silently, or fully hangs the Pydroid 3 app so badly that it must be forcibly killed
and restarted. Whether you get a kill or hang depends on which GUI you run,
but both effects are clearly less than ideal.
<P>
Due to testing constraints, the exact scope of this issue is unknown.
It has been observed on Samsung J7, S8+,
and Note9 devices running Android Nougat, Oreo, and Pie. On the more recent Samsung Note9,
however, rotations in Pydroid 3 tkinter GUIs <I>work fine</I> in both Oreo and Pie after
changing the display's screen-zoom setting from Small to either Medium or Large. This
doesn't help on the older phones, though, and it's unclear if the older phones may benefit
from applying system updates.
<P>
It may also be possible to work around this in Mergeall with a Java-based
<A HREF="https://www.google.com/search?q=android+disable+orientation+change">approach</A>
that locks orientation or ignores rotation, but this would reduce utility drastically
(rotation can be an essential tool when space is limited),
and the underlying bug should really be fixed globally for all programs.
Ultimately, only Pydroid 3 tkinter GUIs exhibit this defect.
<P>
For now: if rotates don't work in tkinter GUIs on your device, either make sure your
screen zoom is not Small and your device is up to date; wait for a fix in whatever
software plays Brutus to Pydroid 3's rotation Caesar;
or use Mergeall's GUI in either portrait or landscape
orientation—<I>only</I>—and consider a rotation-locker app like
<A HREF="https://play.google.com/store/search?q=rotation+lock&c=apps">one of these</A>
if you can't avoid flipping your phone.
<P>
<SPAN class=update>Update</span>: the screen-rotation issue described by
this section <B>has been fixed</B> as of Pydroid 3's 4.X release in 2020.
Specifically, screen rotations have been verified to work well and without
requiring screen-zoom settings when using Pydroid 3 version 4.01 on Samsung
Android versions 7 (Nougat) through 10. Please upgrade your Pydroid 3 app as
needed to avoid this issue.
<DT><A name="_gl4"><A name="rotatedactivities">(Fixed) Phone rotations during activities misplace or hang GUIs</A></A>
<DD>
In the same department as the prior note, changing phone orientation during
a spawned activity may cause a running Pydroid 3 tkinter GUI to slide
off screen, or hang similarly. Slides happen even on phones that
support GUI rotation in general, and have been observed after rotating both
<A HREF="_readme-items/PYDROID3-TKINTER/x-glitches-lostgui-postactivity-1.jpg">Shares</A>
and document handlers opened by the GUI (e.g., web-browser help
<A HREF="_readme-items/PYDROID3-TKINTER/x-glitches-lostgui-postactivity-2.jpg">displays</A>).
Rotations are clearly a Pydroid 3 tkinter problem area; as a workaround,
rotating twice generally restores a slid GUI—but only on devices that support
GUI rotation.
<P>
<SPAN class=update>Update</span>: the screen-rotation issue described by
this section <B>has been fixed</B> as of Pydroid 3's 4.X release in 2020; see
the preceding item's update for more details, and update your Pydroid 3 to avoid
this issue.
<DT><A name="_gl5"><A name="webbrowsermodule">Python's webbrowser works marginally in Pydroid 3's 2.2 release</A></A>
<DD>
Python's <code>webbrowser</code> module for opening web pages and
other content generally works in Pydroid 3, because the app presets
the <code>BROWSER</code> environment variable to an Android <code>am</code>
activity-manager command-line template; when spawned, this template
opens documents per the Android default-apps model. See the module's
<A HREF="https://docs.python.org/3/library/webbrowser.html">documentation</A>
for more on why this works, and
<A HREF="_readme-items/PYDROID3-TKINTER/CODE/_openbrowser.py">this script</A>
for a demo of the technique.
It's similar to <code>open</code> and <code>start</code>
commands on Mac OS and Windows, respectively; on Android, chosen viewers
run in parallel with their opener, and a back tap returns to the
spawner immediately. Documents can also be opened with Python's
<code>os.system</code> and the <code>BROWSER</code> setting,
but that apes what <code>webbrowser</code> does automatically.
<P>
Even so, <code>webbrowser</code> is weak enough on Android to classify as a
glitch (and initially warranted disabling widgets that rely on it).
For one thing, it requires local-file URLs to use a <code>file://</code> prefix
on Android, unlike any other platform—and silently fails to open
if this is absent. For another, Android's default-apps model
is not as functional as file associations on other platforms, and reliance
on this limits <code>webbrowser</code>'s utility: as tested, text files
open with few and unconfigurable handler options, and local HTML files open
as raw HTML in a text editor instead of being rendered by a web browser.
Barring an unknown Android trick, opening HTML in a web browser requires
an <code>http://</code> server URL, which limits HTML documentation to online use only.
<P>
In Mergeall's
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-landscape-1.jpg">GUI</A>, these
limitations impact the "Help" and "Popup logfile?" buttons, which display the
user-guide HTML and run-results text, respectively. These widgets were disabled
in earlier releases requiring users to open documents outside the GUI, but now
work as well as they can. To work around the local-HTML issue, Mergeall now opens its
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-help-html.jpg">user guide</A>
by remote URL; this requires online access, but has the advantage
of always displaying the most recent version (HTML documents tend to change
as frequently as web browsers). To display
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-x-logfile-popup.jpg">logfiles</A>
properly, Mergeall now uses <code>file://</code> prefixes to open per the default apps on
your devices; this may sometimes be too limited to be useful, but it's better than disabling
the feature altogether.
<P>
These workarounds were also adopted on Android by other programs
mentioned in this guide, all of which now open their help and
other documents from within their GUIs, just as they do on PCs.
For coding details, see Mergeall's
<A HREF="#toc83">patched files</A>, as well as those of Frigcal and PyEdit described
<A HREF="https://learning-python.com/using-tkinter-programs-on-android.html#Frigcal">here</A> and
<A HREF="https://learning-python.com/using-tkinter-programs-on-android.html#PyEdit">here</A>.
Caveat: given <code>webbrowser</code>'s limitations, Android users might still be
better served by manually opening documents in some cases.
Caution: per the prior note, Pydroid 3 tkinter GUIs may slide off screen or hang
if phone orientation is changed during a spawned
<A HREF="_readme-items/PYDROID3-TKINTER/x-glitches-lostgui-postactivity-2.jpg">activity</A>;
rotate twice to restore the GUI where
<A HREF="#rotationsglitch">possible</A>.
<P>
<span class=update>Update</span>: this story was rewritten in light of new findings
in mid-April 2019, but was modified yet again by <B>new bugs</B> discovered later in Pydroid 3's
3.0 release—a plot element large enough to warrant its own coverage in the next note.
<DT><A name="_gl6"><A name="webbrowsermodule2.0">Python's webbrowser is fully broken in Pydroid 3's 3.0 release</A></A>
<DD>
<I>Stop the presses</I>—as of early April 2019, Python's <code>webbrowser</code>
module is now fully broken on Android due to changes in release 3.0 of
Pydroid 3. Document opens using this module work as described above in the
prior 2.2 version of Pydroid 3, but they either fail with permission
errors or are deliberately disabled in the latest versions of this app,
and require additional workarounds applied to programs here on April 21.
This complex breakage reflects <I>two</I> separate regressions in Pydroid 3:
<P>
<UL>
<LI>
The app's 3.0 release now sets the <code>DISPLAY</code> environment variable in
tkinter GUIs, which breaks <code>webbrowser</code>—the module assumes an X11 context,
and triggers a permission-error exception when calling tools in Python's
<code>subprocess</code> module to query <code>xdg-settings</code>.
See the new exception
<A HREF="_readme-items/PYDROID3-TKINTER/x-glitches-pydroid3.0-webbrowser.jpg">here</A>.
This happens during browser registration, and before the <code>BROWSER</code> setting
of the next bullet can be applied.
<LI>
The app's 3.0 release has changed its <code>BROWSER</code> environment-variable
setting to run an odd custom script that special-cases
<A HREF="https://jupyter.org/">Jupyter</A> <code>file://</code> documents—but
also senselessly refuses to open any other <code>file://</code> URLs, exiting
silently instead. This setting may help Jupyter documents which bypass
<code>webbrowser</code>, but renders the module unusable for all others.
To see this biased hack for yourself, run <code>echo $BROWSER</code> and
<code>more `which mybrowser`</code> in Pydroid 3 3.0's Terminal;
it breaks other local-file URLs for the sake of a Jupyter kludge,
and appears to do so intentionally.
<!-- <A HREF="_readme-items/PYDROID3-TKINTER/etc/pydroid3-3.0--mybrowser.txt">here</A> -->
</UL>
<P>
Luckily, there are at least two coding workarounds for the damage to other programs:
manually unset <code>DISPLAY</code> to reenable <code>webbrowser</code> in
general <I>and</I> reset <code>BROWSER</code> to its pre-3.0 activity-manager
command for file URLs; or avoid this module altogether by spawning the
activity-manager command with <code>os.system</code> (again, see
<A HREF="_readme-items/PYDROID3-TKINTER/CODE/_openbrowser.py">this demo</A>).
All former <code>webbrowser</code> clients here were
updated to use the latter scheme because it leaves environment
settings intact for other Pydroid 3 use cases.
<P>
<I>Perspective</I>: although this workaround suffices for now, this episode puts the
app's viability as a host for application programs like Mergeall on shakier ground.
The tale of this bug has been rewritten multiple times here already,
and new <code>webbrowser</code> failures introduced by an
"upgrade" of Pydroid 3 that preferentially aids one program at the expense of all
others seem to tip the scales towards unusability—especially given the
lack of notification to paying customers.
At some point, prejudiced instability can be neither documented nor accommodated.
That point may remain uncrossed, but seems clearly in sight in 3.0.
<P>
<span class=update>Update</span>: as of Pydroid 3's 4.0 release,
the <code>DISPLAY</code> environment variable is no longer set,
but the 3.0 Jupyter-biased kludge remains in place, leaving Python's
<code>webbrowser</code> module <B>still broken</B> for all other use cases through
Android 10. Fix, please!
<!-- original: alas, it's easy to be wrong about programs that rely on 5 marginal and shifting systems...
Though less invasive than rotation glitches,
Python's <code>webbrowser</code> module for opening web pages and
other content does not yet support Android.
There may be a Java-based workaround
<A HREF="https://www.google.com/search?q=python+webbrowser+fails+on+android">here too</A>,
but that's a poor option for cross-platform Python code, and waiting for this
...rest cut, incl temp os.system() workaround which just mimics webbrowser+$BROWSER,
but turned out to be necessary after pd3's 3.0 release...
-->
<DT><A name="_gl7">Python's multiprocessing doesn't work on Android (but threads do)</A>
<DD>
While not a Pydroid 3 flaw and irrelevant in Mergeall,
Python's <code>multiprocessing</code> parallel-execution module does not
work on Android, because required semaphore tools are not supported on this
platform
(dig deeper
<A HREF="https://www.google.com/search?q=python+android+multiprocessing">here</A> and
<A HREF="https://en.wikipedia.org/wiki/Android_software_development#Android_NDK">here</A>).
The impact of this will naturally vary per program.
<A HREF="_readme-items/PYDROID3-TKINTER/others-pyedit-grep.png">PyEdit</A>'s
Grep command, for instance, depends on this module on PCs to run searches in parallel
with the GUI; while it can be easily configured to use fully supported threads on
Android instead, doing so makes it impossible for pure Python code to leverage multiple
CPU cores—a speed factor whose significance on phones is unclear.
Moreover, PyEdit benefited from preexisting code that leverages the
API similarity of threads and <code>multiprocessing</code>; other programs
may require more extensive recoding.
<DT><A name="_gl8"><A name="sys.executable">(Fixed) Pydroid 3 breaks spawns by leaving sys.executable empty</A></A>
<DD>
Mergeall's GUI runs the main work of merges as a separate
process by design (this lets you run merges outside the GUI too).
Oddly, Python's <code>sys.executable</code> is left empty by Pydroid 3, which in turn
makes the <code>subprocess.Popen</code> process-spawn code in Mergeall's GUI fail.
Mergeall works around this by hardcoding this setting to use the path to Pydroid 3's
Python, but this should be fixed in Pydroid 3: this same bug crippled most of the
larger programs tested, though some other programs' spawns were also hobbled by the
next issue. The workaround's change also applies to the Pydroid 3 app only, which makes
it impossible to integrate into base files (more on this <A HREF="#toc86">ahead</A>).
<P>
<span class=update>Update</span>: Pydroid 3's 3.0 release in early April 2019 did not
fix its <code>sys.executable</code> bug (this setting is still empty), but it did
<B>move its Python</B> executable to a different path—and break Mergeall's
initial workaround to this bug in the process. See the new exception
<A HREF="_readme-items/PYDROID3-TKINTER/x-glitches-pydroid3.0-pyexepath.jpg">here</A>.
Mergeall's updated workaround addresses this in its April 19
patch-code release, by using the result of a spawned <code>which python</code> shell
command, instead of former or current paths to Pydroid 3 Pythons (see the
<A HREF="_readme-items/PYDROID3-TKINTER/CODE/_whichpy.py">demo code</A>).
The result is release and path agnostic, and will hopefully be immune to future
morph in an app which today seems more likely to sprout packaging-structure
changes than fix reported bugs.
<P>
<span class=update>Update</span>: the Pydroid 3 bug described in this note
<B>has been fixed</B>.
Pydroid 3 doesn't post a formal changes log, but sometime around its 4.0 release in 2020,
this app was fixed to properly set <code>sys.executable</code> to the running
Python interpreter's path in tkinter GUIs. This attribute was left empty in earlier releases,
necessitating the initial hardcoding and later <code>which</code> work-around of the
preceding update. The <code>which</code> work-around is harmless, but is no longer
required—except on devices running older Pydroid 3 versions without the fix.
For an example of how to apply the work-around only when needed, see the updated
<A HREF="_readme-items/PYDROID3-TKINTER/CODE/_whichpy.py">demo code</A>.
<P>
Two postmortem notes about this glitch. First of all, it was later discovered
that Pydroid 3 left <code>sys.executable</code> empty in 2.2 and 3.0 <I>only</I>
for programs that included a statement which <B>imported tkinter</B>, and
were thus identified as GUIs. This was true even if the import statement
was preceded with a <code>#</code> to stub it out as a comment. Programs
without an import, or a comment resembling one, had a correct Python path.
A naive scan for imports is enough to invoke much magic in
Pydroid 3—both dark and otherwise.
<P>
Secondly, though it's impossible to be certain without access to Pydroid 3's
build system, the prior update's <B>Python move</B> in Pydroid 3 3.0
<I>may</I> reflect either Android
<A HREF="https://developer.android.com/ndk/guides">NDK</A> changes,
or the Android app installer's selection of an ABI-specific variant from a fat APK
(i.e., different versions of the compiled code may imply different paths
on different devices).
Regardless of relocation causes, though, the Python interpreter's path should
have been—and now is—reliably stored in <code>sys.executable</code>;
else, a variable path cannot be accommodated without work-around heroics like
those deployed here. More Android ABI details
<A HREF="https://developer.android.com/ndk/guides/abis">here</A> and
<A HREF="https://developer.android.com/ndk/guides/abis#aen">here</A>.
<DT><A name="_gl9"><A name="guiscannotspawnguis">GUIs cannot spawn GUIs</A>
<DD>
Though it doesn't impact Mergeall, tkinter GUI programs can spawn other tkinter GUI
programs in Pydroid 3, but the spawned programs' windows never appear for reasons
unknown. Both
<A HREF="_readme-items/PYDROID3-TKINTER/others-frigcal-launcher.png">Frigcal</A>'s and
<A HREF="_readme-items/PYDROID3-TKINTER/others-pygadgets-launcher.jpg">PyGadgets</A>'
launchers, for instance, start
programs that run (e.g., they can write to files) but remain
<A HREF="_readme-items/PYDROID3-TKINTER/others-pygadgets-noguichildopen.png">invisible</A>,
even though Mergeall's GUI successfully spawns a non-GUI process the
same <code>subprocess</code> way after the prior note's patch.
This is fatal to programs like
<A HREF="https://learning-python.com/pymailgui.html">PyMailGUI</A>
that cannot be used without their
<A HREF="https://learning-python.com/pymailgui-products/unzipped/docetc/docimgs/macosx/composite.png">launchers</A>
and cannot be easily recoded to use top-level windows instead of freestanding processes
(more details
<A HREF="https://learning-python.com/using-tkinter-programs-on-android.html#PyMailGUI">here</A>).
<P>
While its exact cause remains a mystery, this restriction may be inherent
in Pydroid 3 if its tkinter structure requires programs to be run by its IDE.
Tellingly, tkinter programs also fail if run from Pydroid 3's own
<A HREF="_readme-items/PYDROID3-TKINTER/pydroid3-terminal-guis-fail.png">Terminal interface</A>,
and the IDE seems to scan specially for specific import statement forms that
enable GUI support (see the next note).
Moreover, AndroWish—the most likely underlying Tk
<A HREF="#androwish">implementation</A>—appears to have similar
<A href="https://www.androwish.org/index.html/wiki?name=Limitations+of+AndroWish">constraints</A>.
If a workaround exists at all, it is hidden so well that it's effectively nonexistent.
<DT><A name="_gl10">Pydroid 3 is picky about imports</A>
<DD>
Pydroid 3 must see a script's tkinter <code>import</code> or <code>from</code>
statement on a line all by itself, or the script's GUI won't work (the required
scaffolding appears to be unloaded). This didn't require a change in Mergeall,
but the
<A HREF="_readme-items/PYDROID3-TKINTER/others-pytoe.png">PyToe</A> program
in PyGadgets did not open in Pydroid 3 until such a specific statement was
added to its top-level script. Though less common, you may also need to add
a tkinter import statement to programs that load and run tkinter code
from files dynamically; an import in the loaded code won't be enough
to enable tkinter support.
<DT><A name="_gl11">Pydroid 3 crashes badly on larger source files</A>
<DD>
And now for something completely unnecessary:
Pydroid 3's editor has a hard—and arguably naive—limit on the
sizes of files it can open, and won't even warn you when the limit is breached.
Instead, if you open a flawless script whose source file is larger than roughly
256K bytes (or 5K lines), its runs will fail with Android error-message popups stating
that Pydroid 3 has stopped; exiting the script's GUI may exit Pydroid 3 altogether; and the
Pydroid 3 IDE itself may start failing randomly but regularly with the same messages.
It's unclear if bytes or lines kills the IDE, but large files seem its Achilles' heel.
<P>
This surfaced when adding code to the main script of the
<A HREF="https://learning-python.com/pyedit.html">PyEdit</A> program:
a 256,898-byte and 5,141-line prior version worked fine,
but a 258,031-byte and 5,160-line new version started failing for
no reason whatsoever—until wholly unrelated comment lines were removed
(a short launcher script's success was the first clue of size issues).
Given today's memory capacities, this size limit is absurd;
people do write 5k-line program files, and tools do generate them.
It also has no workaround; if your script starts failing inexplicably in
Pydroid 3 after adding new features, make sure its code file is under 256K bytes
(or 5K lines) in size—or write a two-line launcher that reads and
<code>exec</code>s the file to subvert Pydroid 3's draconian constraint.
<DT><A name="_gl12"><A name="doubleback">Double-back exits don't run exit handlers</A></A>
<DD>
Pydroid 3 has a double-back-click option which exits a GUI, and is often
required in <I>maximized</I> viewing mode due to the absence of a window-border
exit button (the top-right corner's "X" in windows like
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-nonmax-nonfull-portrait.jpg">this</A>).
Strangely, double-back
exits close GUIs immediately without running their exit handlers. While such
silent exits have little consequence to users in Mergeall, they can be
catastrophic in other programs that must finalize work on closes,
and are at best a use-at-your-own-peril feature.
<P>
The <A HREF="https://learning-python.com/pyedit.html">PyEdit</A>
text editor, for example, has an explicit Quit button that works normally, but has
no way to offer users a chance to save file
<A HREF="_readme-items/PYDROID3-TKINTER/others-pyedit-save.jpg">changes</A>
on double-back exits. Worse, the
<A HREF="https://learning-python.com/frigcal.html">Frigcal</A> calendar has
no explicit quit widget of its own, and hence no way to save calendars in
maximized mode. Non-maximized mode does better with a window-border exit button
that runs exit handlers properly, but doesn't use available space as well, and can
still be the scene of a silent-but-deadly double-back exit. The lesson here:
don't use a double-back in Pydroid 3 unless doing nothing is what you mean to do.
<P>
<span class=update>Update</span>: Pydroid 3's 4.0 release in 2020 seems to
have changed double-back exits to be <B>single-back</B> exits: just one
system-back tap now suffices to close a GUI on Samsung devices tested.
This seems inadvertent (it's still dubbed "double" in option menus), and may
be a new buglet; either way, exit handlers are still skipped on the exit—perilously.
<DT><A name="_gl13">Newlines may need help</A>
<DD>
Though not a factor in Mergeall, pressing the return (a.k.a. enter) key on some
Samsung on-screen keyboards does not move the cursor to a new line in
text widgets of tkinter GUIs in Pydroid 3. For reasons unknown,
the return key's newline character never makes the trip between keyboard
and widget, which renders general text-input components virtually unusable.
This bug may be limited to specific versions of the Samsung keyboard—it's
been observed only when using version 3.2 on Android Oreo, and both older and
newer versions' return keys work correctly. But where returns fail, they
appear to fail in Pydroid 3 tkinter GUIs exclusively.
<P>
Luckily, there are at least three ways to work around this glitch if it impacts your device:
use a physical Bluetooth keyboard; use Samsung's handwriting input option whose return key works correctly;
or simply install an alternative on-screen keyboard whose return key works properly, such as
<A HREF="https://play.google.com/store/apps/details?id=com.google.android.inputmethod.latin">Google's Gboard</A>
or
<A HREF="https://play.google.com/store/apps/details?id=org.pocketworkstation.pckeyboard">Hacker's Keyboard</A>.
Both alternative keyboards support multiline-text input in programs like
<A HREF="https://learning-python.com/pyedit.html">PyEdit</A> and
<A HREF="https://learning-python.com/frigcal.html">Frigcal</A>,
and the latter keyboard also helps with the next glitch on this list.
<DT><A name="_gl14">Keypresses may need help</A>
<DD>
Though this also doesn't impact Mergeall, and isn't a bug in any of the systems
involved, PC GUIs designed to expect a keyboard may require redesign or logistical
workarounds when used in Pydroid 3, because the on-screen keyboard may not be
readily available. The
<A HREF="https://learning-python.com/pygadgets.html">PyPhoto</A> program,
for example, requires keypresses to process photos, and its main display has no
text input field to tap—the only way to force most preinstalled on-screen
keyboards to appear.
<P>
To work around this, use a Bluetooth keyboard,
or install an alternative on-screen keyboard on your phone that can be opened on demand.
<A HREF="https://play.google.com/store/apps/details?id=org.pocketworkstation.pckeyboard">Hacker's Keyboard</A>,
for instance, includes a permanent-notification option which allows you to pop up its on-screen
keyboard at any time by clicking the app's entry in Android's
<A HREF="_readme-items/PYDROID3-TKINTER/x-notific-keyboard-termux.jpg">notifications</A>
pulldown.
The
<A HREF="_readme-items/PYDROID3-TKINTER/x-glitches-pyphoto-hackerskbrd.jpg">resulting keyboard</A>
is a complete solution in PyPhoto, and helps with newlines, menu shortcuts, and copy/paste in other
programs too (see the notes above and below).
<DT><A name="_gl15">Keypresses may need silencing</A>
<DD>
Of course, too much of a good thing can also be a bad thing: Android's automatic on-screen
keyboard popup may be an <I>unwelcome</I> feature in some Pydroid 3 tkinter GUIs.
Interacting with any text field mindlessly triggers the
popup—unless it's silenced by code changes which mark text fields as uneditable,
with Tk's <code>readonly</code> or <code>disabled</code> states.
<P>
The
<A HREF="https://learning-python.com/frigcal.html">Frigcal</A> program, for example,
required <code>readonly</code> state in Tk
<code>entry</code>
<A HREF="https://www.tcl.tk/man/tcl8.4/TkCmd/entry.htm#M11">widgets</code></A>
<!-- true, but selects are iffy, and too much detail here: "along with code to negate shifts," -->
to mute annoying keyboard popups on most touch gestures that made the program unusable.
Mergeall's GUI wasn't impacted quite as badly, but its message-text
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-portrait-6.jpg">area</A> still required
read-only state on Android (which means <code>disabled</code> in Tk's
<A HREF="https://www.tcl.tk/man/tcl8.4/TkCmd/text.htm#M12"><code>text</code></A>);
otherwise, slower swipes to scroll trigger an on-screen keyboard
popup—which overlays and obscures the very text you're trying to view.
Despite the work, keyboard popups may still appear more often than they should in
some Pydroid 3 tkinter GUIs; as usual on Android, your device's back button (and a
downswipe where supported) hides an on-screen keyboard that's obscuring important
content or GUI components.
<DT><A name="_gl16"><A name="keypresshandlers">Keypresses botch shifts in some programs</A></A>
<DD>
A technical keyboard issue not relevant to Mergeall but a potential pitfall to others:
Pydroid 3's tkinter also fails to send expected details to general keypress
handlers for all physical and on-screen keyboards tested. For instance, the return
and backspace keys send tkinter event objects with an empty <code>char</code> attribute
and a <code>keysym</code> that identifies the press (as <code>'Return'</code>
and <code>'BackSpace'</code>, respectively). This differs from other platforms
but can be worked around with binds to these specific keys
(e.g., to <code><Return></code>).
<P>
Worse, though: <I>shifted</I> keys, like <code>+</code> and <code>*</code>
on most keyboards, send two useless keypress events—one for the shift
key itself, and another for the <I>unshifted</I> key pressed. A
<code>Shift+*</code> sequence, for example, sends a shift followed by the
unshifted <code>8</code> key only; the <code>*</code> never comes through.
Binding to specific keys doesn't help in these cases, and mapping unshifted
to shifted keys manually seems implausible across all keyboards and usage contexts.
Barring a workaround as yet unknown, this makes shifted keys unusable in some
tkinter GUIs run on Pydroid 3.
<P>
This glitch seems unrelated to the newlines issue noted earlier (which fails
on just one keyboard), and on the upside appears limited to general keyboard
handlers (tkinter text-input widgets process shifts correctly). But it impacts
programs like
<A HREF="https://learning-python.com/pygadgets.html">PyCalc</A>
that allow numeric operator keys to be entered on keyboards. As is,
PyCalc fully supports keyboards in its "cmd" popup input field
(whose use also seems to oddly correct the shifted-keys bug), but keyboard
input for its main display is limited to non-shifted keys on Android
<A HREF="https://learning-python.com/using-tkinter-programs-on-android.html#pycalckeyboards">alone</A>.
<DT><A name="_gl17">Copy/paste may need help</A>
<DD>
Android-style copy and paste via longpresses is not available in tkinter GUIs
run in Pydroid 3. Instead, to copy and paste text between any text-input widgets:
first, highlight text with touch, stylus, or mouse; then either use program-specific widgets
(e.g., menu options) where available to copy and paste, or use the PC-like key combinations
<code>Ctrl+c</code> to copy and <code>Ctrl+v</code> to paste in keyboards that support
these. You'll find support for these keys in both Bluetooth keyboards, as well as
alternative on-screen keyboards like
<A HREF="https://play.google.com/store/apps/details?id=org.pocketworkstation.pckeyboard">Hacker's Keyboard</A>
(yes, a pattern is emerging). This works well, but may not be what a typical Android user
would expect.
<DT><A name="_gl18"><A name="buttonissues">Buttons glitch #1: losing background color</A></A>
<DD>
The fundamental Button widget in Pydroid 3's tkinter has a pair of issues
that seem to reflect state problems and do not respond to tkinter
<code>update</code> calls. Only one impacts Mergeall and the other is
narrow in scope, but both merit a heads-up for both users and programmers.
<P>
Most noticeably, on every Android device tested, Buttons lose
their <code>bg</code> background-color setting when pressed: their color changes
to the recessed default, and stays that way until anything else on the
GUI is tapped. This can seem to make a Button
<A HREF="_readme-items/PYDROID3-TKINTER/x-glitches-pycalc-buttons-color.png">appear pressed</A>,
but is really just background colors being lost. Curiously, some buttons evade
color loss (e.g., those that open common dialogs may be immune, suggesting a
timing issue), but most colored buttons are impacted.
<P>
This colors glitch affects only
the "Help" button in Mergeall, but may afflict other GUIs more broadly. The only known
workaround requires changing code to either accept the Button's default color,
or replace Buttons with Label widgets that use click bindings.
Labels retain their background
colors when clicked, and their cosmetic and functional difference is trivial
(Labels don't give press feedback, but Buttons in Pydroid 3 sometimes
don't either).
As fix examples, Mergeall eventually chose default
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-landscape-1.jpg">colors</A>; the
<A HREF="https://learning-python.com/pygadgets.html">PyCalc</A>
calculator program adopted
<A HREF="_readme-items/PYDROID3-TKINTER/others-pycalc.jpg">Labels</A> to
<A HREF="https://learning-python.com/using-tkinter-programs-on-android.html#buttonsstuckon">skirt the issue</A>;
and the
<A HREF="https://learning-python.com/frigcal.html">Frigcal</A> and
<A HREF="https://learning-python.com/pygadgets.html">PyToe</A>
programs have used Labels all along
(since their ports to Mac OS, which has similar button-color limitations).
<DT><A name="_gl19">Buttons glitch #2: getting stuck</A>
<DD>
More rarely, Buttons may actually become stuck in a pressed
state, in which they remain
<A HREF="_readme-items/PYDROID3-TKINTER/x-glitches-other-buttons-stuck.png">visibly recessed</A>,
and may respond to a future press by triggering two press events at once.
Luckily, this glitch appears to be very limited in scope; it has been observed on
only one device running Android Nougat (a Samsung S8+) and only occasionally on this
device, and has not been seen on any other devices tested—including
Samsung devices running Oreo and Pie, and an older Samsung J7 also running
Nougat. The flaw trigger on the S8+ is a complete unknown (like much in
Android forensics), and there is no known workaround. If your tkinter
buttons are truly stuck "on," tap elsewhere in the GUI to "unpress" them.
<DT><A name="_gl20">Android issues cast a wide net</A>
<DD>
Although not specific to Pydroid 3, the Android issues we've seen besetting
Mergeall afflict other tkinter GUIs too. In the
<A HREF="https://learning-python.com/frigcal.html">Frigcal</A> calendar GUI,
for instance, calendar updates are limited to internal storage or app-specific
folders on removable drives, and work only on Android Oreo and later due
to the same timestamp bug that constrains <A HREF="#toc21">Mergeall</A>.
Violating these rules produces the respective crash scenes
<A HREF="_readme-items/PYDROID3-TKINTER/others-frigcal-fail-termuxappdir.jpg">here</A> and
<A HREF="_readme-items/PYDROID3-TKINTER/others-frigcal-fail-nougattime.png">here</A>.
In Frigcal, the timestamp bug kills calendar backups and makes it view-only
before Oreo. In other GUIs, Android's bugs and permission rules
may limit functionality similarly.
<DT><A name="_gl21">And so on</A>
<DD>
This appendix cannot provide a comprehensive list of Pydroid 3 tkinter
defects—and it's impossible to report them to lists elsewhere,
or even reference such lists, when the underlying implementation is
<A HREF="#androwish">unknown</A>—but testing performed so far
has uncovered additional pitfalls which merit brief cautions here.
In the other-issues bucket, Pydroid 3's tkinter also:
<P>
<UL>
<!-- now a larger writeup above...
<LI>
Appears to leave buttons stuck "on" in
<A HREF="_readme-items/PYDROID3-TKINTER/x-glitches-pycalc-buttons-color.png">some GUIs</A>,
perhaps due to lost
<A HREF="https://learning-python.com/using-tkinter-programs-on-android.html#buttonsstuckon">background colors</A>
-->
<LI>
Munges Unicode symbols in non-maximized window
<A HREF="_readme-items/PYDROID3-TKINTER/x-glitches-pyedit-botched-unicode.jpg">titles</A>,
and sometimes on
<A HREF="https://learning-python.com/using-tkinter-programs-on-android.html#pyeditsymbols">buttons</A>
<LI>
Whether feature or anomaly, draws a weird
<A HREF="_readme-items/PYDROID3-TKINTER/x-glitches-pyedit-weird-rectangle.jpg">rectangle</A>
around the cursor in
<A HREF="https://learning-python.com/using-tkinter-programs-on-android.html#weirdrectangles">some contexts</A>
<LI>
Truncates instead of scrolling <code>OptionMenu</code> selection lists too large for the
<A HREF="_readme-items/PYDROID3-TKINTER/x-glitches-options-list-truncated.png">display</A>
<LI>
Badly truncates text in common
<A HREF="_readme-items/PYDROID3-TKINTER/x-giltches-dialogs-truncate-text.jpg">dialogs</A>,
instead of wrapping it like
<A HREF="https://learning-python.com/mergeall-products/unzipped/docetc/docimgs/macosx/run-confirm.png">every</A>
<A HREF="https://learning-python.com/mergeall-products/unzipped/docetc/docimgs/windows/run-confirm.png">other</A>
<A HREF="https://learning-python.com/mergeall-products/unzipped/docetc/docimgs/linux/run-confirm.png">platform</A>
<LI>
Posts fullscreen windows at display top
where they are apt to argue with Android's status-bar
<A HREF="_readme-items/PYDROID3-TKINTER/x-glitches-statusbar-conflict.png">pulldown</A>
<LI>
Does not recognize the <code>Alt</code> key for menu shortcuts, in
<A HREF="https://play.google.com/store/apps/details?id=org.pocketworkstation.pckeyboard">keyboards</A>
that support it
<LI>
Uses Linux file- and folder-chooser
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-landscape-2.jpg">dialogs</A> that can be tedious
to use on phones, and may be best
<A HREF="https://learning-python.com/using-tkinter-programs-on-android.html#mergeallpathconfigs">avoided</A>
<LI>
Fails to dispatch stylus-hover
<A HREF="https://learning-python.com/using-tkinter-programs-on-android.html#footers">events</A>,
though mouse hovers work fine (and fingers don't hover)
<LI>
As noted <A HREF="#guiusage">earlier</A>, uses rightclick
triggers that seem unusual, and is limited to running one GUI at a time
<LI>
As also <A HREF="#Pydroid 3 Viewing Modes">noted</A>, has a maximized
mode more suited for single-window apps than multi-window
<A HREF="https://learning-python.com/using-tkinter-programs-on-android.html#moreonfit">programs</A>
<LI>
May truncate windows—and even flash spastically—if the
navigation bar is hidden while running a GUI
<LI>
Has been known to fire multiple events for a single gesture
<LI>
Requires an occasional restart to restore competence
<LI>
And crashes without clear cause more commonly than it should
</UL>
<P>
Probably the best spin on these is symptoms of still-maturing software.
</DL>
<!-- or is spasmodically funnier? (but spasmically isn't a word) -->
<P>
There's more on the other programs noted in the preceding list in the next section.
The point here is that while some accommodation is to be expected when
translating between PC and smartphone paradigms, issues like those above
mean that running tkinter programs in Pydroid 3 is hardly seamless.
Even allowing for fixes in its future releases, you'll clearly want
to test-drive this app on tkinter programs you'd like to use to see how
its support applies.
<DIV class=notebox>
<P class=noteboxheader>
Later Glitches Off-Page
<P class=noteboxlast>
<SPAN class=update>Update</SPAN>:
Later testing on later Androids uncovered new and additional
Pydroid 3 tkinter glitches—including <B>doubled characters</B> for shifted keys
(on both Android 10 and 11 Samsung devices, and perhaps all Pydroid 3 4.0);
and permission errors when spawning <B>activity-manager commands</B> that start intents
(just in Pydroid 3 on Android 11). Android 11 also revokes USB-drive access
and slows to a crawl programs that process files in shared storage (like Mergeall),
but these are not Pydroid 3's responsibility.
For the whole story on these later glitches, see the
<A HREF="https://learning-python.com/mergeall-android11-updates.html#android11strikesback">Android 11 updates page</A>.
</P>
</DIV>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc86>The Advertising Scourge</A></H3>
<P>
Despite the workarounds and glitches of the preceding section,
the second to last of the Pydroid 3 requirements listed earlier is really its most
regrettable and grievous downside: the free version of Pydroid 3 will flash
distracting and distasteful fullscreen ads in your face every few runs, unless
and until you pay a required $9.99
<A HREF="https://www.google.com/search?q=define+freemium">freemium</A> fee.
<!-- oxford dictionary's page has photos of people; sigh -->
To be blunt, this comes off as ransom, and borders on extortion. It may be a
norm in Android's culture, but it is rude and unacceptable in Python's,
and makes it difficult to recommend Pydroid 3 to Mergeall users today.
<P>
Together with the prior section's glitches, this also makes it impossible to
integrate the changes required for Pydroid 3 tkinter GUIs into source-code
package base files. For example, the coding workarounds that accommodate
Pydroid 3's font limitations and <code>webbrowser</code> breakage are specific
to Pydroid 3 alone, and the <code>sys.executable</code> fix required
for spawns may not work in other apps.
Changing portable source-code packages
to support just a <I>single app</I> on Android is ethically dubious at
best—and just plain wrong when that app employs advertising.
Though less convenient, Pydroid 3 today is best supported
with the custom source-code files used here.
<P>
This guide's broader advice to Mergeall's users is to evaluate its GUI in
Pydroid 3 to see if it warrants the fee in your context. It may, especially
if you have other tkinter programs you wish to run on Android. As examples
of this app's broader scope:
<UL>
<LI>
The smaller tkinter tutorial examples in the
<A HREF="https://learning-python.com/about-pp4e.html">book</A>
Programming Python, 4th Edition work unchanged in Pydroid 3, like
<A HREF="_readme-items/PYDROID3-TKINTER/others-pp4e-1.png">this</A>,
<A HREF="_readme-items/PYDROID3-TKINTER/others-pp4e-2.png">this</A>,
<A HREF="_readme-items/PYDROID3-TKINTER/others-pp4e-3.png">this</A>,
<!-- <A HREF="_readme-items/PYDROID3-TKINTER/others-pp4e-4.jpg">this</A>, -->
<A HREF="_readme-items/PYDROID3-TKINTER/others-pp4e-5.png">this</A>, and
<A HREF="_readme-items/PYDROID3-TKINTER/others-pp4e-6.png">this</A>,
though some require path settings for imports that can be
forced with Python <code>sys.path</code> appends.
<LI>
Other major tkinter
applications available at Mergeall's hosting
<A HREF="https://learning-python.com/programs.html">site</A>—including
<A HREF="_readme-items/PYDROID3-TKINTER/others-frigcal-1.jpg">Frigcal</A>,
<A HREF="_readme-items/PYDROID3-TKINTER/others-pyedit.png">PyEdit</A>, and the
PyGadgets utilities
<A HREF="_readme-items/PYDROID3-TKINTER/others-pycalc.jpg">PyCalc</A>,
<A HREF="_readme-items/PYDROID3-TKINTER/others-pyclock.png">PyClock</A>,
<A HREF="_readme-items/PYDROID3-TKINTER/others-pyphoto.png">PyPhoto</A>, and
<A HREF="_readme-items/PYDROID3-TKINTER/others-pytoe.png">PyToe</A>—also
run as shown in Pydroid 3 with minimal changes similar to Mergeall's, but benefit more
from tkinter support than Mergeall because they lack command-line modes.
</UL>
<P>
See <A HREF="https://learning-python.com/using-tkinter-programs-on-android.html"><I>this page</I></A>
for a separate how-to on running these programs on Android too, along with a summary of
this appendix's Mergeall GUI coverage. Taken as a set, programs like these
and others may justify the ransom's price, but this choice is yours to make.
<P>
Wherever you may land on this issue, there's no denying that Pydroid 3's
advertising scheme casts programs like Mergeall in a supporting sales role
that's both uncomfortable and unseemly. A free trial period would be much
more palatable than obtrusive advertising, and will hopefully be a part of
the future of the otherwise-remarkable Pydroid 3 app.
Barring that, we'll have to eagerly await the Android debut of free and
open-source alternatives for running the free and open-source tkinter.
<DIV class=notebox>
<P class=noteboxheader>
It's a Large World After All
<P class=noteboxlast>
Naturally, there may be other tkinter-on-Android options afoot or
unknown at this guide's publication, and others may appear in the
future. Please exercise the usual due diligence, and check back
here for updates as change warrants and time allows. This guide
wishes to be inclusive, but some commercial projects may be no more
transparent than they are benevolent, and the quality of web-search
results has grown inversely proportional to the avarice of companies
providing them.
</P>
</DIV>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc87><A name=pydroidpermissions>General Requirements Still Apply</A></A></H3>
<P>
Finally, in addition to all the Pydroid 3 requirements described in the preceding
sections, you must also keep in mind that this appendix's GUI approach shares all
the other general constraints that this guide notes for the Termux command-lines
approach. Specifically, to use Mergeall's GUI in Pydroid 3, you'll still need to:
<UL>
<LI><A HREF="#toc21">Use</A> Oreo 8 and higher only, due to a prior Android timestamp bug
<LI><A HREF="#toc22">Use</A> FAT32 eternal drives on some devices, due to a former Samsung exFAT bug
<LI><A HREF="#toc23">Use</A> Mergeall's source-code package, because that's what Pydroid 3 runs too
<LI><A HREF="#toc24">Grant</A> Pydroid 3 extra storage permissions, requested automatically in this app
<LI><A HREF="#toc25">Nest</A> updateable content on removable drives in Pydroid 3's app-specific folder
<LI><A HREF="#toc26">Beware</A> of losing content nested per the prior bullet if you uninstall Pydroid 3
</UL>
<P>
In other words, Mergeall's GUI may run on Android, but it's still running
on Android.
<DIV class=notebox>
<P class=noteboxheader>
More on Pydroid 3 Storage Permissions
<P>
Notice the last half of the general requirements listed in this section—it arises because
Pydroid 3 shares the exact same permission constraints that we saw <A HREF="#toc24">earlier</A>
for Termux. Namely, the programs that Pydroid 3 runs can <I>read</I> anywhere on internal storage
and removable media, and can <I>write</I> anywhere on internal storage, but <I>writes</I> on
removable media are limited to Pydroid 3's app-specific folder only, which is used
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-to-sdcard.jpg">like this</A>, and lives
at the following path on your <code>/storage/xxxx-xxxx</code> drive (once you create it,
and swap the <code>x</code>s for your drive's <A HREF="#toc7">Android ID</A>):
<P>
<!-- don't overflow mobile viewport, else entire doc scrolls -->
<!-- the break-word allows breaks anywhere if overflows -->
<!-- a compromise; PRE doesn't work here, and P breaks flow-->
<!-- overflow-wrap doesn't seem to avoid all overflows in iOS Safari (a bug?) -->
<!-- see android-deltas-sync/_README.html note for more on the diff -->
<!-- use <code>'s word-wrap: break-all here (and ahead too) -->
<!-- drop <space style="margin-left: 12px"> to avoid breaks -->
<code>Android/data/ru.iiec.pydroid3</code>
<P>
It's easy to verify this: see the permissions
<A HREF="_readme-items/PYDROID3-TKINTER/CODE/_permissions.py">test script</A>,
and its results on both
<A HREF="_readme-items/PYDROID3-TKINTER/permissions-results-pydroid3.png">Pydroid 3</A> and
<A HREF="_readme-items/PYDROID3-TKINTER/permissions-results-termux.png">Termux</A>;
the results do not differ between Android Nougat and
<A HREF="_readme-items/PYDROID3-TKINTER/permissions-results-pydroid3-oreo.jpg">Oreo</A>
and are unchanged by installing Pydroid 3's extra-permissions
<A HREF="https://play.google.com/store/apps/details?id=ru.iiec.pydroidpermissionsplugin">plugin</A>.
The takeaway from these results is that storage access rules are the <I>same</I> for both apps,
and updates on removable media are limited in both to the app's <I>own folder</I>. The only
real difference in the storage-permissions department is that Pydroid 3 replaces Termux's
one-time <A HREF="#toc32">script</A> run with a one-time automatic request on first access
to restricted folders.
<P>
As we've <A HREF="#toc25">learned</A>,
this means that content on <I>removable media</I> must be nested in Pydroid 3's
app-specific folder if it is to be updated by Mergeall in this app. It also means that items
to be updated by <I>both</I> Termux and Pydroid 3 must be located in <I>internal storage</I>,
because these apps can modify only their own app-specific folders on removable media.
Importantly, your choice of Mergeall content folder on SD cards and USB drives wholly
determines which access-limited app can change it. For instance, if Mergeall's GUI in
Pydroid 3 processes removable-drive content in Termux folders—as in earlier
<A HREF="_readme-items/PYDROID3-TKINTER/mergeall-landscape-1.jpg">screenshots</A>—it
is limited to comparisons
<A HREF="_readme-items/PYDROID3-TKINTER/CODE/_permissions-cross-app.py">only</A>.
And while our focus here is on Mergeall, this
constraint applies to any GUI run in the
<A HREF="_readme-items/PYDROID3-TKINTER/others-frigcal-fail-termuxappdir.jpg">app</A>.
<P>
This is an unavoidable downside to using Android removable storage in these apps.
The lesson here is that
<I>internal storage</I>—usually at <code>/sdcard</code>—should
always be used if possible, because it can be changed by both Termux and Pydroid 3
specifically, and by other apps generally. As an example of the latter set,
<A HREF="https://play.google.com/store/apps/details?id=org.qpython.qpy3">QPython3</A>,
a Python Android app not covered in this guide because
it holds no advantages for Mergeall today, has the same storage-access limits per
<A HREF="_readme-items/PYDROID3-TKINTER/permissions-results-qpython3.png">this</A> and
<A HREF="_readme-items/PYDROID3-TKINTER/qpython-permissions.png">this</A>;
Android's rules must be hard to beat.
<P class=noteboxlast>
<SPAN class=update>Update</SPAN>:
though uncertain at this writing, <B>Android 11</B> may impose
the same access constraints on internal storage that KitKat imposed on removable
media, and invalidate some of this section's coverage in the process.
See <A HREF="#thefutureisclosed">Appendix C</A> for more on the possible
consequences for Mergeall—and every other program that saves shared
data on the Android platform.
</P>
</DIV>
<!-- ---------------------------------------------------------------------------- -->
<H3><A name=toc88>The Mergeall Pydroid 3 Wrap-Up</A></H3>
<P>
So there it is: the good, the bad, and the ugly of using tkinter GUIs
like Mergeall's in the Pydroid 3 Android app. To be sure, the tkinter support
embedded in this app is nothing short of <I>stunning</I>: it implements nearly
all of Tk and tkinter on smartphones, and makes these devices substantially more
fun to use, for both programmers and others. You'll have to decide for yourself
whether its remaining defects—along with the advertising model of the app
that brings it to you—justifies using this support on your phone.
<P>
Beyond Pydroid 3's tkinter support, there are additional GUI options for
Python programs on Android that we don't have space to cover here.
For example, while Termux does not offer tkinter support directly, it
does provide some X Windows (a.k.a. X11) GUI support that may provide a path
to using tkinter in its Linux environment. For a survey, see its links
<A HREF="https://wiki.termux.com/wiki/Graphical_Environment">here</A> and
<A HREF="https://github.com/termux/x11-packages">here</A>, and the X11 search
<A HREF="https://duckduckgo.com/?q=android+x11">here</A>.
Even if viable, however, X11 support requires substantial extra setup
work that may be too much to ask of users running Mergeall alone.
<P>
Other GUI options offer Android-friendlier alternatives, including
<A HREF="https://en.wikipedia.org/wiki/Kivy_(framework)">Kivy</A>,
<A HREF="https://en.wikipedia.org/wiki/Scripting_Layer_for_Android">SL4A</A>, and Qt's
<A HREF="https://www.riverbankcomputing.com/software/pyqt/intro">PyQt</A> and
<A HREF="https://www.qt.io/qt-for-python">PySide</A>.
Such tools allow programs to use Python for their core logic, but
wrap it in GUIs which may better match the Android paradigm. Although
their program recoding costs are not trivial, these tools become more
attractive for tkinter GUIs that must be redesigned for smaller screens if
their audiences are ever to include more than the most-stubborn tkinter
<A HREF="https://mail.python.org/pipermail/tkinter-discuss/2014-September/003668.html">fans</A>.
<P style="margin-bottom: 30px";>
At the end of the day, Pydroid 3's tkinter support is admirable and even
amazing, but the approach it offers for Mergeall really just swaps command
lines for a GUI run in an IDE that requires extra code changes and comes with
noticeable seams. Whether you find it easier to sync your content
by running a single command in Termux, or opening and running a source-code
file in the Pydroid 3 IDE may very well depend on your background.
For this guide's money, though, the latter is just enough extra steps
to qualify as runner-up.
<!-- ============================================================================ -->
<H2><A name=toc9><A name="thefutureisclosed">Appendix C: Android 11—the Future is Closed?</A></A></H2>
<P class=preface>
<SPAN class=preface>Preface</SPAN>:
This appendix was written when the changes it describes were planned for Android Q.
After publication, these changes were pushed out to Android R, and Androids Q
and R were rebranded 10 and 11—both respectively and confusingly. Android's
names have been updated here, but the rest of this story was left intact; its
potentially paralyzing changes still loom large for apps and users in Android 11.
<P>
And now for the closing cliff-hanger. True to its thrashing-laden history,
Android seems poised to radically alter storage-access permissions again
and soon, with potentially profound impacts on both Mergeall and the Android
platform in general. Although this change is still tentative, this appendix
provides a preview of its implications for users of Android and Mergeall in
one possible future.
<H3><A name="toc91">The Permissions Game Changer</A></H3>
<P>
This how-to has gone to great lengths to document the permission
rules for internal and removable storage that existed at the time
of publication, especially in sections
<A HREF="#toc24">here</A>,
<A HREF="#toc25">here</A>, and
<A HREF="#toc87">here</A>. As covered, these rules apply to apps
running on Androids KitKat through Pie, of which only Oreo and Pie
(and later, 10) are actually useable for Mergeall due to an earlier
Android timestamps <A HREF="#toc21">bug</A>.
<P>
Regrettably, some of this guide's coverage of the rules for
<I>internal storage</I>—the unremovable storage space in
your phone that much Android documentation <!--confusingly--> misleadingly
calls external—might
be invalidated in the near future. It now appears that Android 10,
released in beta form in March 2019 (when it was known as Q), may impose the
same crippling constraints on internal storage that Android KitKat earlier imposed
on removable media.
In brief, this so-called <I>scoped storage</I> proposal would restrict access
to internal storage as follows:
<UL>
<LI>
An app would have free access only to files it created in either its
own sandboxed folder or a common media folder
<LI>
All the files in an app's sandboxed folder would be automatically
and silently deleted when the app is uninstalled
<LI>
Files in common media folders may survive app uninstalls,
but could be freely accessed only by their creating app
<LI>
The portable and standard POSIX file API would not be directly
usable for content outside an app's own sandboxed folder
</UL>
<!--
Nit: the POSIX API may be usable at the native level after going through
the SAF to fetch a file descriptor, but this dilutes the point too much:
(update: no, it matters: standalone apps may grab file descriptors in
Java and pass to C++ for NTK POSIX usage; this is still whack with
poo-brain, but may be possible, especially in a standalone app):
<LI>
The portable and standard POSIX file API would not be directly
usable for files outside an app's own sandboxed folder
-->
<P>
The app <I>sandboxed</I> folder referenced in this list is what we've called the
<I>app-specific</I> folder (e.g., in <code>/sdcard/Android</code>) in this
<A HREF="#toc7">document</A>, because its internal-storage version can be
accessed by other apps today. Apps already have an <I>app-private</I> folder
(e.g., in <code>/data/data</code>), but the new rules would limit apps' access
to their own app-specific folder in the currently general internal storage.
<P>
This model would be the death knell of same-device
interoperability: because files in internal storage would no longer
be freely usable across multiple apps, the scope of your on-phone content
would be limited to a single role. This is fully unusable
for modern computing tasks.
By sandboxing apps, the proposed Android
<B><I>severs the data workflow at the heart of content development</I></B>.
It also manages to further trample the ethic of
engineering freedom promoted for decades by the
open-source world; a platform that imposes tools and rules on this scale
would be closed by definition.
<P>
Some of the new constraints may be mitigated by wildly proprietary and
convoluted interfaces, but these interfaces may be much slower than those of
traditional <A HREF="https://en.wikipedia.org/wiki/POSIX">POSIX</A> files,
and are unlikely to see widespread use in
programs born on less restrictive systems and designed to run on multiple
platforms. Either way, this change would impact nearly every program that
updates shared content on Android devices (including Mergeall), and could
very well be an endgame for this platform's broader potential.
<P>
For more background on this proposed change than this appendix can provide,
read the terse and perpetually morphing announcement
<!-- arbitrarily renamed by mid 2020...
<A HREF="https://developer.android.com/preview/privacy/scoped-storage">here</A>;
-->
<A HREF="https://developer.android.com/preview/privacy/storage">here</A>;
the pragmatic technical review by an Android file-explorer developer
<A HREF="https://www.xda-developers.com/android-q-storage-access-framework-scoped-storage/">here</A>;
and the outrage of Android developers pleading
<!-- and petitioning --> for a reversal
<A HREF="https://issuetracker.google.com/issues/128591846">here</A>,
<A HREF="https://www.reddit.com/r/androiddev/comments/b10bpo/android_q_new_scoped_storage_question/">here</A>, and
<A HREF="https://issuetracker.google.com/issues/131580997">here</A>
(the Mergeall team is guilty of the latter).
<!--
<A HREF="https://issuetracker.google.com/issues/128599055">here</A>
-->
<!--(and starting petitions
<A HREF="https://www.change.org/p/google-inc-stop-google-from-implementing-scoped-storage-in-android?recruiter=947522148&utm_source=share_petition&utm_medium=copylink&utm_campaign=share_petition">here</A>).
-->
<P>
Caution: the language at some of those links is understandably explicit.
If this change becomes permanent—and KitKat's example suggests
it will—it threatens to make a host of existing programs and libraries based
on the portable POSIX file standard unusable on Android; render Android's internal
storage just as locked down and limited as that in iOS;
grossly curtail what users can do with their content on their phones;
and effectively kill the platform as an option for general-purpose computing.
<P>
Simply put, under the scoped-storage model, Android devices may be reduced to
launchers of fenced-in apps, and lose all rights to the title of "PC alternative."
Indeed, the better analogy for the proposed Android seems the cartridge-based
Game Boy of eras
<A HREF="https://en.wikipedia.org/wiki/Game_Boy">past</A>.
<H3><A name="toc92">The Impact on Mergeall</A></H3>
<P>
That being said, Mergeall itself will likely still work unchanged on Android
if these restrictions are imposed, but content on internal storage may have to
be structured in the same inconvenient and perilous way as that on removable
media. Specifically, it might need to be nested in the updating app's
app-specific folder in internal storage. This means the following
sandboxed locations for the Termux and Pydroid 3 apps covered
in this guide, respectively:
<PRE style="line-height: 125%;">
/sdcard/Android/data/com.termux/MY-STUFF
/sdcard/Android/data/ru.iiec.pydroid3/MY-STUFF
</PRE>
<P>
Unfortunately, content thus stored could generally be updated only by the <I>sole
app</I> whose folder hosts its storage, and would be automatically <I>deleted</I>
if that app is ever uninstalled. While these major constraints already
apply to removable media today, extending them to internal storage may
make the Mergeall program impractical for many or most users on Android.
<P>
There's a slim chance that the more permanent <code>Download</code>
<I>shared-collection</I> folder in internal storage would still be usable
for Mergeall content in the Termux and Pydroid 3 apps. That is,
you may be able to locate your content in internal storage just
one level deeper, at:
<PRE>
/sdcard/Download/MY-STUFF
</PRE>
<P>
If viable, this might avoid the delete-on-uninstall peril of sandboxed folders,
and may support access from multiple apps. Also unfortunately, the beta release's
<!--
<A HREF="https://developer.android.com/preview/privacy/scoped-storage#media-files">documentation</A>
-->
<!--
<A HREF="https://developer.android.com/preview/privacy/scoped-storage#filtered-view">documentation</A>
-->
<A HREF="https://developer.android.com/preview/privacy/storage#file-directory-restrictions">documentation</A>
of proposed rules for the <code>Download</code> folder suggests that files stored
there may be app specific too, though this is described thinly and ambiguously, and will
have to await empirical analysis on widely used devices.
<P>
In fact, the very definition of "access" to be restricted is unclear in the beta documents,
and also awaits verification in the final release.
For example, if the new internal-storage rules shadow those of removable media
<A HREF="#toc25">today</A>, apps may still be able to <I>read</I> anywhere
in internal storage, leaving a massive security hole completely unfilled by the new model.
But shared-storage <I>writes</I>—essential for content creation of any
significance—are almost certain to vanish, at least in their current direct form.
<P>
As a last resort, the beta also suggests using Android's
<A HREF="https://developer.android.com/guide/topics/providers/document-provider">Storage Access Framework</A>
(SAF) to access files created by other apps (e.g., by passing file descriptors down to the
<!--
<A HREF="https://developer.android.com/preview/privacy/scoped-storage#media-from-native-code">native code</A>
-->
<A HREF="https://developer.android.com/preview/privacy/storage#media-direct-file-native">native code</A>
layer that hosts Python on <A HREF="https://developer.android.com/ndk">Android</A>).
But this proprietary library is Java based, unwieldly, and slow; may not work for
Mergeall's mass-updates use case; and seems a nonstarter for cross-platform Python
code run by an app which sets permissions.
Python programs use the POSIX file interface on purpose: it makes them portable
to every modern computing platform of interest—except the proposed Android.
For much code, the extreme requirements of the SAF would make this platform
unsupportable.
<P>
In the end, Mergeall and similarly general tools may run on Android 10,
but they're likely to be much more difficult and dubious to use, if not
altogether pointless. Content collections that are accessible to just one
app and may be subject to accidental removal would probably not be
worth storing on your phone.
<H3><A name="toc93">The Impact on Android</A></H3>
<P>
Naturally, it's impossible to predict if internal-storage restrictions
will turn out to be anything more than beta-release drama, and you'll
have to stay tuned for this story's conclusion.
There is substantial developer pushback at present; apps like Termux
and Pydroid 3 that run Python code may implement broader storage
permissions and interfaces in response;
phone manufacturers may implement internal storage in ways that
cannot be foreseen;
<!--
internal storage might take
different forms on different phones that cannot be foreseen;
-->
and an initially proposed
<!--
<A HREF="https://developer.android.com/preview/privacy/scoped-storage#compatibility-mode">
-->
<I>compatibility mode</I>
promised to retain the current internal-storage permissions
of apps used in this guide—at least until those apps targeted
later releases, you uninstalled the apps, or you bought a new phone.
<P>
What <I>is</I> certain is that deliberately hobbling the Android
platform is sure to limit both its users' options, and its devices'
scope. Extending proprietary lockdown to internal storage would
make portable tools like Mergeall nearly unusable on Android, and
constrain the platform to artificially narrow—and even toyish—apps.
Such apps might appeal to technically naive masses and generate
proportionate advertising revenue in the process, but the closed platform
hosting these apps will hold little interest for the skilled development
community that produces nontrivial programs used elsewhere.
<P>
The net effect would deprive smartphone users of much useful software,
and knowingly retard their devices' utility. This would be especially
ironic at a time when foldable displays and ample storage hold the
promise of expanded roles.
Though more subtle and speculative, this change may also help drive
smartphone users to cloud-based storage in the face of unworkable on-phone
alternatives—boosting subscriptions, trackable content, and
single-source control as darker byproducts.
In this analysis, Android would remain a carefully crafted arena
of exploitation, fueled by greed and justified by red-herring security
arguments easily refuted by the platform's own success.
<P>
Let's hope that's not what Android's owner has in mind.
</P>
<H3><A name="toc94">Updates: The Rest of the Story</A></H3>
<P>
Because Android's internal-storage permissions were still an early and
evolving epic when this guide was written, this section has been added
to provide updates that arose after publication. Some of its content may be
developer oriented, but its twists seem worthy of dramatists of all kinds.
<div class=notebox>
<P class=noteboxheader>
Moved Off-Page
<P>
After being amended twice, this updates section grew too
unwieldly for inline appearance here, and has been split off to this
<A HREF="https://learning-python.com/mergeall-android11-updates.html"><B>separate page</B></A>.
Please visit that page for the continuing saga of Python and tkinter
on Android 11 and later.
<P class=noteboxlast>
<I>Spoiler</I>: in Android 11+, the All Files Access permission restores
general file-path and POSIX access to both shared storage and USB drives,
and the Storage Access Framework provides API-based access to single folders.
Of these, the former comes at the expense of Play-store presence for
some types of apps; the latter is much more an Android-only solution;
and both generally require standalone apps. The future may not have
turned out to be fully closed, but it did become much more proprietary.
<!--
no.. this would have to stay in sync:
To go to individual updates directly, click the following links:
<P>
<UL>
<LI><A HREF="#mergeal-android-11-updates.htmlcrisispostponed">Crisis Postponed until 11 (Apr-2019)</A>
<LI><A HREF="#mergeal-android-11-updates.htmlallfilesaccess">A New Hope—All Files Access (Oct-2020)</A>
<LI><A HREF="#mergeal-android-11-updates.htmlandroid11strikesback">The Empire Strikes Back... (Jan-2021) </A>
</UL>
-->
</P>
</DIV>
<!-- ============================================================================ -->
<!-- killed after navbar added
<P class=summary style="border-top: thin solid black;">
<BR>
<I>Have feedback on this guide?—send <A HREF="mailto:lutz@learning-python.com">email</A></I>.
<BR>
<I>See also: Mergeall's <A HREF="https://learning-python.com/mergeall.html">homepage</A>
and <A HREF="https://learning-python.com/mergeall-products/unzipped/UserGuide.html">user guide</A>,
and other <A HREF="https://learning-python.com/programs.html">programs</A>.
</P>
-->
<!------------------------------------------------------------------------------------------------>
</P>
<BR><BR>
<DIV class=footerdiv>
<TABLE class=footertable>
<!--borrowed from websites' theme-->
<TD align=center>
<A class=blocklinkbar href="https://learning-python.com/index.html"> <!--border=0 for IE img links-->
<IMG SRC="_readme-items/PythonPowered.gif" ALT="[Python Logo]" border=0 width=55 height=22></A> <!--scale larger gif for res-->
<!-- Nov19: top link iff no JS (else floating top button) -->
<noscript>
<TD align=center>
<B><A class=blocklinkbar HREF="#">Top</A></B>
</noscript>
<TD align=center>
<B><A class=blocklinkbar HREF=".">Code</A></B>
<!-- Nov19: since we have space if JS (and nearly always will) -->
<TD align=center>
<B><A class=blocklinkbar HREF="_readme-items/index.html">Media</A></B>
<TD align=center>
<B><A class=blocklinkbar HREF="https://learning-python.com/using-tkinter-programs-on-android.html">Tkinter</A></B>
<TD align=center>
<B><A class=blocklinkbar HREF="https://learning-python.com/mergeall.html">Mergeall</A></B>
<TD align=center>
<B><A class=blocklinkbar HREF="https://learning-python.com/post-release-updates.html">News</A></B>
<TD align=center>
<B><A class=blocklinkbar HREF="https://learning-python.com/posts.html">Blog</A></B>
<TD align=center>
<B><A class=blocklinkbar HREF="https://learning-python.com/programs.html">Apps</A></B>
<TD align=center>
<B><A class=blocklinkbar HREF="mailto:lutz@learning-python.com">Input</A></B>
<!--
<TD align=center style="color: white;">
©<I>M.Lutz</I>
-->
<TD align=center>
<A class=blocklinkbar HREF="https://learning-python.com/formalbio.html">©<I>M.Lutz</I></A>
</TABLE>
</DIV>